2024-04-26T04:31:14.830 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T04:31:14.830 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573 --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --verbose --name teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/orig.config.yaml 2024-04-26T04:31:14.889 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T04:31:14.976 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7674573' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 8320 sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 10/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a targets: smithi005.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK1FqpcbwxuFR2PR6h4paKtpuL+OgrR0eRI4VMHgod6Ly1CTALJy/DKRXKkuUkYBZb2N+iosSnC+rhv1gPf4ez8= smithi111.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCtaZHKumg0BsyIabyssGu8Zd23naBvygtR/DUBfKwLs7FaLfGaYfWH39ViSjc84K2xG8q+ZNaoi/CzKgZDHdgc= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-26T04:31:14.976 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa; will attempt to use it 2024-04-26T04:31:14.977 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks 2024-04-26T04:31:14.977 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T04:31:14.979 INFO:teuthology.task.internal:Checking packages... 2024-04-26T04:31:15.002 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a' 2024-04-26T04:31:15.002 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:31:15.003 INFO:teuthology.packaging:ref: None 2024-04-26T04:31:15.003 INFO:teuthology.packaging:tag: None 2024-04-26T04:31:15.003 INFO:teuthology.packaging:branch: squid 2024-04-26T04:31:15.003 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:31:15.003 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-26T04:31:15.348 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2447.gb22e2ebd 2024-04-26T04:31:15.350 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T04:31:15.357 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T04:31:15.357 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T04:31:15.371 INFO:teuthology.task.internal:Saving configuration 2024-04-26T04:31:15.392 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T04:31:15.399 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:31:15.423 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi005.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:25:42.611804', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK1FqpcbwxuFR2PR6h4paKtpuL+OgrR0eRI4VMHgod6Ly1CTALJy/DKRXKkuUkYBZb2N+iosSnC+rhv1gPf4ez8='} 2024-04-26T04:31:15.459 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi111.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:25:42.610383', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCtaZHKumg0BsyIabyssGu8Zd23naBvygtR/DUBfKwLs7FaLfGaYfWH39ViSjc84K2xG8q+ZNaoi/CzKgZDHdgc='} 2024-04-26T04:31:15.459 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T04:31:15.466 INFO:teuthology.task.internal:roles: ubuntu@smithi005.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-26T04:31:15.466 INFO:teuthology.task.internal:roles: ubuntu@smithi111.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-26T04:31:15.466 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T04:31:15.537 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f89fd2460d0>, signals=[15]) 2024-04-26T04:31:15.537 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T04:31:15.543 INFO:teuthology.task.internal:Opening connections... 2024-04-26T04:31:15.543 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi005.front.sepia.ceph.com 2024-04-26T04:31:15.545 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi005.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:31:15.624 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi111.front.sepia.ceph.com 2024-04-26T04:31:15.624 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi111.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:31:15.700 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T04:31:15.707 DEBUG:teuthology.orchestra.run.smithi005:> uname -m 2024-04-26T04:31:15.725 INFO:teuthology.orchestra.run.smithi005.stdout:x86_64 2024-04-26T04:31:15.726 DEBUG:teuthology.orchestra.run.smithi005:> cat /etc/os-release 2024-04-26T04:31:15.784 INFO:teuthology.orchestra.run.smithi005.stdout:NAME="CentOS Stream" 2024-04-26T04:31:15.784 INFO:teuthology.orchestra.run.smithi005.stdout:VERSION="9" 2024-04-26T04:31:15.784 INFO:teuthology.orchestra.run.smithi005.stdout:ID="centos" 2024-04-26T04:31:15.784 INFO:teuthology.orchestra.run.smithi005.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:VERSION_ID="9" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:ANSI_COLOR="0;31" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:31:15.785 INFO:teuthology.orchestra.run.smithi005.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:31:15.786 INFO:teuthology.lock.ops:Updating smithi005.front.sepia.ceph.com on lock server 2024-04-26T04:31:15.810 DEBUG:teuthology.orchestra.run.smithi111:> uname -m 2024-04-26T04:31:15.828 INFO:teuthology.orchestra.run.smithi111.stdout:x86_64 2024-04-26T04:31:15.828 DEBUG:teuthology.orchestra.run.smithi111:> cat /etc/os-release 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:NAME="CentOS Stream" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:VERSION="9" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:ID="centos" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:VERSION_ID="9" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:ANSI_COLOR="0;31" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:31:15.885 INFO:teuthology.orchestra.run.smithi111.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:31:15.886 INFO:teuthology.orchestra.run.smithi111.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:31:15.886 INFO:teuthology.orchestra.run.smithi111.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:31:15.886 INFO:teuthology.lock.ops:Updating smithi111.front.sepia.ceph.com on lock server 2024-04-26T04:31:15.910 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T04:31:15.920 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T04:31:15.925 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T04:31:15.925 DEBUG:teuthology.orchestra.run.smithi005:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:31:15.928 DEBUG:teuthology.orchestra.run.smithi111:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:31:15.943 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T04:31:16.002 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T04:31:16.003 DEBUG:teuthology.orchestra.run.smithi005:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:31:16.005 DEBUG:teuthology.orchestra.run.smithi111:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:31:16.074 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T04:31:16.127 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T04:31:16.141 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-26T04:31:16.142 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-26T04:31:16.142 DEBUG:teuthology.orchestra.run.smithi005:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:31:16.144 DEBUG:teuthology.orchestra.run.smithi111:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:31:16.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:31:16.161 DEBUG:teuthology.orchestra.run.smithi111:> uname -r 2024-04-26T04:31:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:31:16.162 DEBUG:teuthology.orchestra.run.smithi005:> uname -r 2024-04-26T04:31:16.218 INFO:teuthology.orchestra.run.smithi111.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:31:16.218 INFO:teuthology.task.kernel:Running kernel on smithi111: 5.14.0-437.el9.x86_64 2024-04-26T04:31:16.218 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum install -y kernel 2024-04-26T04:31:16.220 INFO:teuthology.orchestra.run.smithi005.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:31:16.221 INFO:teuthology.task.kernel:Running kernel on smithi005: 5.14.0-437.el9.x86_64 2024-04-26T04:31:16.221 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum install -y kernel 2024-04-26T04:31:20.225 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - BaseOS 3.5 MB/s | 8.0 MB 00:02 2024-04-26T04:31:20.487 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - BaseOS 3.1 MB/s | 8.0 MB 00:02 2024-04-26T04:31:23.942 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - AppStream 7.2 MB/s | 19 MB 00:02 2024-04-26T04:31:23.989 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - AppStream 8.0 MB/s | 19 MB 00:02 2024-04-26T04:31:28.668 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - CRB 5.3 MB/s | 6.2 MB 00:01 2024-04-26T04:31:29.930 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - CRB 2.8 MB/s | 6.2 MB 00:02 2024-04-26T04:31:30.590 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - Extras packages 27 kB/s | 16 kB 00:00 2024-04-26T04:31:31.786 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - Extras packages 25 kB/s | 16 kB 00:00 2024-04-26T04:31:33.298 INFO:teuthology.orchestra.run.smithi005.stdout:Extra Packages for Enterprise Linux 15 MB/s | 21 MB 00:01 2024-04-26T04:31:38.588 INFO:teuthology.orchestra.run.smithi005.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-04-26T04:31:40.494 INFO:teuthology.orchestra.run.smithi005.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:31:40.494 INFO:teuthology.orchestra.run.smithi005.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:31:40.599 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:31:40.599 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:31:40.599 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:31:40.712 DEBUG:teuthology.orchestra.run.smithi005:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:31:41.272 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:31:38 AM UTC. 2024-04-26T04:31:41.395 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout:Reinstalling: 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:31:41.404 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:31:41.405 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 255 k 2024-04-26T04:31:41.405 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 0 2024-04-26T04:31:41.405 INFO:teuthology.orchestra.run.smithi005.stderr:Operation aborted. 2024-04-26T04:31:41.469 INFO:teuthology.orchestra.run.smithi005.stdout:Is this ok [y/N]: 2024-04-26T04:31:41.469 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum reinstall -y kernel || true 2024-04-26T04:31:41.984 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:31:38 AM UTC. 2024-04-26T04:31:42.105 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout:Reinstalling: 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:31:42.106 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:31:42.107 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:31:42.107 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 255 k 2024-04-26T04:31:42.107 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 0 2024-04-26T04:31:42.107 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:31:42.389 INFO:teuthology.orchestra.run.smithi005.stdout:kernel-5.14.0-437.el9.x86_64.rpm 2.0 MB/s | 255 kB 00:00 2024-04-26T04:31:42.390 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:42.390 INFO:teuthology.orchestra.run.smithi005.stdout:Total 898 kB/s | 255 kB 00:00 2024-04-26T04:31:42.406 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:31:42.614 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:31:42.614 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:31:42.619 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:31:42.619 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:31:42.748 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:31:42.824 INFO:teuthology.orchestra.run.smithi005.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:31:42.824 INFO:teuthology.orchestra.run.smithi005.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:31:43.468 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:31:43.468 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:31:44.194 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:31:44.194 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:31:44.194 INFO:teuthology.orchestra.run.smithi005.stdout:Reinstalled: 2024-04-26T04:31:44.194 INFO:teuthology.orchestra.run.smithi005.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:31:44.194 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:31:44.195 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:31:44.380 DEBUG:teuthology.orchestra.run.smithi005:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:31:44.428 INFO:teuthology.orchestra.run.smithi005.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:31:44.429 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:31:44.429 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:31:44.429 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-26T04:31:44.429 DEBUG:teuthology.orchestra.run.smithi005:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:31:44.457 INFO:teuthology.orchestra.run.smithi005.stdout:ttyS1 2024-04-26T04:31:44.488 DEBUG:teuthology.parallel:result is None 2024-04-26T04:31:46.240 INFO:teuthology.orchestra.run.smithi111.stdout:Extra Packages for Enterprise Linux 1.3 MB/s | 21 MB 00:15 2024-04-26T04:31:51.381 INFO:teuthology.orchestra.run.smithi111.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-04-26T04:31:53.296 INFO:teuthology.orchestra.run.smithi111.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:31:53.296 INFO:teuthology.orchestra.run.smithi111.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:31:53.391 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:31:53.392 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:31:53.392 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:31:53.496 DEBUG:teuthology.orchestra.run.smithi111:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:31:53.990 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:02 ago on Fri 26 Apr 2024 04:31:51 AM UTC. 2024-04-26T04:31:54.112 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:31:54.126 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:31:54.126 INFO:teuthology.orchestra.run.smithi111.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout:Reinstalling: 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 255 k 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 0 2024-04-26T04:31:54.127 INFO:teuthology.orchestra.run.smithi111.stderr:Operation aborted. 2024-04-26T04:31:54.177 INFO:teuthology.orchestra.run.smithi111.stdout:Is this ok [y/N]: 2024-04-26T04:31:54.178 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum reinstall -y kernel || true 2024-04-26T04:31:54.669 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:31:51 AM UTC. 2024-04-26T04:31:54.784 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout:Reinstalling: 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:31:54.785 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 255 k 2024-04-26T04:31:54.786 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 0 2024-04-26T04:31:54.786 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:31:56.067 INFO:teuthology.orchestra.run.smithi111.stdout:kernel-5.14.0-437.el9.x86_64.rpm 226 kB/s | 255 kB 00:01 2024-04-26T04:31:56.067 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:56.068 INFO:teuthology.orchestra.run.smithi111.stdout:Total 199 kB/s | 255 kB 00:01 2024-04-26T04:31:56.092 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:31:56.287 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:31:56.287 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:31:56.292 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:31:56.292 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:31:56.439 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:31:56.552 INFO:teuthology.orchestra.run.smithi111.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:31:56.552 INFO:teuthology.orchestra.run.smithi111.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:31:57.226 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:31:57.226 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:31:57.915 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:31:57.916 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:31:57.916 INFO:teuthology.orchestra.run.smithi111.stdout:Reinstalled: 2024-04-26T04:31:57.916 INFO:teuthology.orchestra.run.smithi111.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:31:57.916 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:31:57.916 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:31:58.079 DEBUG:teuthology.orchestra.run.smithi111:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:31:58.126 INFO:teuthology.orchestra.run.smithi111.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:31:58.126 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:31:58.126 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:31:58.126 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-26T04:31:58.126 DEBUG:teuthology.orchestra.run.smithi111:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:31:58.161 INFO:teuthology.orchestra.run.smithi111.stdout:ttyS1 2024-04-26T04:31:58.199 DEBUG:teuthology.parallel:result is None 2024-04-26T04:31:58.199 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T04:31:58.206 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T04:31:58.207 DEBUG:teuthology.orchestra.run.smithi005:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:31:58.210 DEBUG:teuthology.orchestra.run.smithi111:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:31:58.253 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T04:31:58.263 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T04:31:58.275 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T04:31:58.275 DEBUG:teuthology.orchestra.run.smithi005:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:31:58.277 DEBUG:teuthology.orchestra.run.smithi111:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:31:58.318 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T04:31:58.328 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T04:31:58.328 DEBUG:teuthology.orchestra.run.smithi005:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:31:58.341 DEBUG:teuthology.orchestra.run.smithi111:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:31:58.374 INFO:teuthology.orchestra.run.smithi005.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:31:58.382 INFO:teuthology.orchestra.run.smithi111.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:31:58.385 INFO:teuthology.orchestra.run.smithi005.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:31:58.391 INFO:teuthology.orchestra.run.smithi111.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:31:58.392 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T04:31:58.402 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T04:31:58.402 DEBUG:teuthology.orchestra.run.smithi005:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:31:58.429 DEBUG:teuthology.orchestra.run.smithi111:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:31:58.456 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T04:31:58.466 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T04:31:58.467 DEBUG:teuthology.orchestra.run.smithi005:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:31:58.498 DEBUG:teuthology.orchestra.run.smithi111:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:31:58.513 DEBUG:teuthology.orchestra.run.smithi005:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:31:58.600 DEBUG:teuthology.orchestra.run.smithi005:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:31:58.648 DEBUG:teuthology.orchestra.run.smithi005:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:31:58.734 DEBUG:teuthology.orchestra.run.smithi005:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:31:58.774 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:31:58.774 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:31:58.842 DEBUG:teuthology.orchestra.run.smithi111:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:31:58.879 DEBUG:teuthology.orchestra.run.smithi111:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:31:58.928 DEBUG:teuthology.orchestra.run.smithi111:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:31:59.011 DEBUG:teuthology.orchestra.run.smithi111:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:31:59.049 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:31:59.049 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:31:59.111 DEBUG:teuthology.orchestra.run.smithi005:> sudo service rsyslog restart 2024-04-26T04:31:59.114 DEBUG:teuthology.orchestra.run.smithi111:> sudo service rsyslog restart 2024-04-26T04:31:59.148 INFO:teuthology.orchestra.run.smithi005.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:31:59.190 INFO:teuthology.orchestra.run.smithi111.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:31:59.555 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T04:31:59.564 INFO:teuthology.task.internal:Starting timer... 2024-04-26T04:31:59.564 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T04:31:59.579 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T04:31:59.592 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-26T04:31:59.593 DEBUG:teuthology.orchestra.run.smithi005:> sudo service auditd rotate 2024-04-26T04:31:59.643 INFO:teuthology.orchestra.run.smithi005.stdout:Rotating logs: 2024-04-26T04:31:59.645 DEBUG:teuthology.orchestra.run.smithi111:> sudo service auditd rotate 2024-04-26T04:31:59.700 INFO:teuthology.orchestra.run.smithi111.stdout:Rotating logs: 2024-04-26T04:31:59.702 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T04:31:59.702 DEBUG:teuthology.orchestra.run.smithi005:> /usr/sbin/getenforce 2024-04-26T04:31:59.721 INFO:teuthology.orchestra.run.smithi005.stdout:Permissive 2024-04-26T04:31:59.721 DEBUG:teuthology.orchestra.run.smithi111:> /usr/sbin/getenforce 2024-04-26T04:31:59.745 INFO:teuthology.orchestra.run.smithi111.stdout:Permissive 2024-04-26T04:31:59.745 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi005.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi111.front.sepia.ceph.com': 'permissive'} 2024-04-26T04:31:59.745 DEBUG:teuthology.orchestra.run.smithi005:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:31:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:31:59.787 DEBUG:teuthology.orchestra.run.smithi111:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:31:59.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:31:59.812 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T04:31:59.812 DEBUG:teuthology.orchestra.run.smithi005:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:31:59.854 DEBUG:teuthology.orchestra.run.smithi111:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:31:59.881 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T04:31:59.891 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-26T04:31:59.897 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-26T04:32:00.026 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T04:32:00.041 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T04:32:00.047 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi005.front.sepia.ceph.com,smithi111.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T04:35:32.395 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi005.front.sepia.ceph.com'), Remote(name='ubuntu@smithi111.front.sepia.ceph.com')] 2024-04-26T04:35:32.396 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi005.front.sepia.ceph.com' 2024-04-26T04:35:32.397 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi005.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:35:32.479 DEBUG:teuthology.orchestra.run.smithi005:> true 2024-04-26T04:35:32.513 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi005.front.sepia.ceph.com' 2024-04-26T04:35:32.513 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi111.front.sepia.ceph.com' 2024-04-26T04:35:32.514 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi111.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:35:32.595 DEBUG:teuthology.orchestra.run.smithi111:> true 2024-04-26T04:35:32.631 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi111.front.sepia.ceph.com' 2024-04-26T04:35:32.631 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T04:35:32.641 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T04:35:32.641 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:35:32.642 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:35:32.644 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:35:32.644 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:35:32.671 INFO:teuthology.orchestra.run.smithi005.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:35:32.685 INFO:teuthology.orchestra.run.smithi005.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:35:32.698 INFO:teuthology.orchestra.run.smithi111.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:35:32.708 INFO:teuthology.orchestra.run.smithi005.stderr:sudo: ntpd: command not found 2024-04-26T04:35:32.711 INFO:teuthology.orchestra.run.smithi111.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:35:32.718 INFO:teuthology.orchestra.run.smithi005.stdout:506 Cannot talk to daemon 2024-04-26T04:35:32.732 INFO:teuthology.orchestra.run.smithi005.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:35:32.733 INFO:teuthology.orchestra.run.smithi111.stderr:sudo: ntpd: command not found 2024-04-26T04:35:32.743 INFO:teuthology.orchestra.run.smithi111.stdout:506 Cannot talk to daemon 2024-04-26T04:35:32.745 INFO:teuthology.orchestra.run.smithi005.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:35:32.756 INFO:teuthology.orchestra.run.smithi111.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:35:32.770 INFO:teuthology.orchestra.run.smithi111.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:35:32.806 INFO:teuthology.orchestra.run.smithi005.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:35:32.810 INFO:teuthology.orchestra.run.smithi005.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:35:32.810 INFO:teuthology.orchestra.run.smithi005.stdout:=============================================================================== 2024-04-26T04:35:32.810 INFO:teuthology.orchestra.run.smithi005.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.810 INFO:teuthology.orchestra.run.smithi005.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.810 INFO:teuthology.orchestra.run.smithi005.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.810 INFO:teuthology.orchestra.run.smithi005.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.822 INFO:teuthology.orchestra.run.smithi111.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:35:32.826 INFO:teuthology.orchestra.run.smithi111.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:35:32.826 INFO:teuthology.orchestra.run.smithi111.stdout:=============================================================================== 2024-04-26T04:35:32.826 INFO:teuthology.orchestra.run.smithi111.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.826 INFO:teuthology.orchestra.run.smithi111.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.826 INFO:teuthology.orchestra.run.smithi111.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.826 INFO:teuthology.orchestra.run.smithi111.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:35:32.827 INFO:teuthology.run_tasks:Running task install... 2024-04-26T04:35:32.835 DEBUG:teuthology.task.install:project ceph 2024-04-26T04:35:32.836 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'}} 2024-04-26T04:35:32.836 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:35:32.836 INFO:teuthology.task.install:Using flavor: default 2024-04-26T04:35:32.843 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-26T04:35:32.844 INFO:teuthology.task.install:extra packages: [] 2024-04-26T04:35:32.844 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:35:32.844 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:35:32.844 INFO:teuthology.packaging:ref: None 2024-04-26T04:35:32.844 INFO:teuthology.packaging:tag: None 2024-04-26T04:35:32.844 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:35:32.844 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:35:32.845 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:35:32.846 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:35:32.847 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:35:32.847 INFO:teuthology.packaging:ref: None 2024-04-26T04:35:32.847 INFO:teuthology.packaging:tag: None 2024-04-26T04:35:32.847 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:35:32.847 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:35:32.847 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:35:33.073 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:35:33.075 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:35:33.167 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:35:33.167 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:35:33.204 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:35:33.204 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:35:33.204 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:35:33.229 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:35:33.229 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:35:33.229 INFO:teuthology.packaging:ref: None 2024-04-26T04:35:33.230 INFO:teuthology.packaging:tag: None 2024-04-26T04:35:33.230 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:35:33.230 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:35:33.230 DEBUG:teuthology.orchestra.run.smithi111:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:35:33.279 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:35:33.279 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:35:33.279 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:35:33.295 DEBUG:teuthology.orchestra.run.smithi111:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:35:33.307 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:35:33.307 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:35:33.307 INFO:teuthology.packaging:ref: None 2024-04-26T04:35:33.307 INFO:teuthology.packaging:tag: None 2024-04-26T04:35:33.307 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:35:33.307 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:35:33.307 DEBUG:teuthology.orchestra.run.smithi005:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:35:33.369 DEBUG:teuthology.orchestra.run.smithi111:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:35:33.373 DEBUG:teuthology.orchestra.run.smithi005:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:35:33.433 INFO:teuthology.orchestra.run.smithi111.stdout:check_obsoletes = 1 2024-04-26T04:35:33.434 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum clean all 2024-04-26T04:35:33.450 DEBUG:teuthology.orchestra.run.smithi005:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:35:33.516 INFO:teuthology.orchestra.run.smithi005.stdout:check_obsoletes = 1 2024-04-26T04:35:33.518 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum clean all 2024-04-26T04:35:33.658 INFO:teuthology.orchestra.run.smithi111.stdout:48 files removed 2024-04-26T04:35:33.686 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-radosgw 2024-04-26T04:35:33.752 INFO:teuthology.orchestra.run.smithi005.stdout:48 files removed 2024-04-26T04:35:33.778 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-radosgw 2024-04-26T04:35:34.357 INFO:teuthology.orchestra.run.smithi111.stdout:ceph packages for x86_64 187 kB/s | 77 kB 00:00 2024-04-26T04:35:34.379 INFO:teuthology.orchestra.run.smithi005.stdout:ceph packages for x86_64 234 kB/s | 77 kB 00:00 2024-04-26T04:35:34.790 INFO:teuthology.orchestra.run.smithi111.stdout:ceph noarch packages 27 kB/s | 11 kB 00:00 2024-04-26T04:35:34.821 INFO:teuthology.orchestra.run.smithi005.stdout:ceph noarch packages 27 kB/s | 11 kB 00:00 2024-04-26T04:35:35.116 INFO:teuthology.orchestra.run.smithi111.stdout:ceph source packages 6.1 kB/s | 1.8 kB 00:00 2024-04-26T04:35:35.164 INFO:teuthology.orchestra.run.smithi005.stdout:ceph source packages 5.8 kB/s | 1.8 kB 00:00 2024-04-26T04:35:35.767 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.0 MB 00:00 2024-04-26T04:35:35.852 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-04-26T04:35:38.253 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - AppStream 14 MB/s | 19 MB 00:01 2024-04-26T04:35:38.301 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - AppStream 14 MB/s | 19 MB 00:01 2024-04-26T04:35:42.462 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - CRB 9.0 MB/s | 6.2 MB 00:00 2024-04-26T04:35:42.594 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.2 MB 00:00 2024-04-26T04:35:44.176 INFO:teuthology.orchestra.run.smithi005.stdout:CentOS Stream 9 - Extras packages 45 kB/s | 16 kB 00:00 2024-04-26T04:35:44.303 INFO:teuthology.orchestra.run.smithi111.stdout:CentOS Stream 9 - Extras packages 23 kB/s | 16 kB 00:00 2024-04-26T04:35:45.177 INFO:teuthology.orchestra.run.smithi005.stdout:Extra Packages for Enterprise Linux 24 MB/s | 21 MB 00:00 2024-04-26T04:35:45.823 INFO:teuthology.orchestra.run.smithi111.stdout:Extra Packages for Enterprise Linux 15 MB/s | 21 MB 00:01 2024-04-26T04:35:50.610 INFO:teuthology.orchestra.run.smithi005.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-04-26T04:35:51.045 INFO:teuthology.orchestra.run.smithi111.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-04-26T04:35:52.891 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:35:52.893 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:35:52.893 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:35:52.893 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:35:52.893 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:35:52.893 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:35:52.893 INFO:teuthology.orchestra.run.smithi005.stdout:Upgrading: 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:35:52.894 INFO:teuthology.orchestra.run.smithi005.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:35:52.895 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:35:52.896 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:35:52.896 INFO:teuthology.orchestra.run.smithi005.stdout:Install 22 Packages 2024-04-26T04:35:52.896 INFO:teuthology.orchestra.run.smithi005.stdout:Upgrade 2 Packages 2024-04-26T04:35:52.896 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:35:52.896 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 58 M 2024-04-26T04:35:52.896 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:35:53.169 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout:Upgrading: 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:35:53.171 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:35:53.172 INFO:teuthology.orchestra.run.smithi111.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout:Install 22 Packages 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout:Upgrade 2 Packages 2024-04-26T04:35:53.173 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:35:53.174 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 58 M 2024-04-26T04:35:53.174 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:35:53.986 INFO:teuthology.orchestra.run.smithi005.stdout:(1/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86_ 7.7 MB/s | 5.0 MB 00:00 2024-04-26T04:35:54.036 INFO:teuthology.orchestra.run.smithi005.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 505 kB/s | 25 kB 00:00 2024-04-26T04:35:54.154 INFO:teuthology.orchestra.run.smithi005.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 5.3 MB/s | 637 kB 00:00 2024-04-26T04:35:54.220 INFO:teuthology.orchestra.run.smithi005.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 6.7 MB/s | 457 kB 00:00 2024-04-26T04:35:54.837 INFO:teuthology.orchestra.run.smithi005.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 8.0 MB/s | 4.9 MB 00:00 2024-04-26T04:35:54.921 INFO:teuthology.orchestra.run.smithi005.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 542 kB/s | 45 kB 00:00 2024-04-26T04:35:55.013 INFO:teuthology.orchestra.run.smithi005.stdout:(7/24): python3-ceph-common-17.2.7-904.gd8c6b0a 1.3 MB/s | 117 kB 00:00 2024-04-26T04:35:55.113 INFO:teuthology.orchestra.run.smithi005.stdout:(8/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 1.5 MB/s | 156 kB 00:00 2024-04-26T04:35:55.221 INFO:teuthology.orchestra.run.smithi005.stdout:(9/24): python3-rados-17.2.7-904.gd8c6b0a3.el9. 2.9 MB/s | 320 kB 00:00 2024-04-26T04:35:55.322 INFO:teuthology.orchestra.run.smithi005.stdout:(10/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 2.9 MB/s | 293 kB 00:00 2024-04-26T04:35:55.414 INFO:teuthology.orchestra.run.smithi005.stdout:(11/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.1 MB/s | 100 kB 00:00 2024-04-26T04:35:55.522 INFO:teuthology.orchestra.run.smithi005.stdout:(12/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.4 MB/s | 160 kB 00:00 2024-04-26T04:35:55.548 INFO:teuthology.orchestra.run.smithi005.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.8 MB/s | 45 kB 00:00 2024-04-26T04:35:55.614 INFO:teuthology.orchestra.run.smithi005.stdout:(14/24): librdkafka-1.6.1-102.el9.x86_64.rpm 9.8 MB/s | 662 kB 00:00 2024-04-26T04:35:55.793 INFO:teuthology.orchestra.run.smithi005.stdout:(15/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9. 5.7 MB/s | 14 MB 00:02 2024-04-26T04:35:55.825 INFO:teuthology.orchestra.run.smithi005.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.4 MB/s | 292 kB 00:00 2024-04-26T04:35:55.842 INFO:teuthology.orchestra.run.smithi005.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 857 kB/s | 42 kB 00:00 2024-04-26T04:35:55.891 INFO:teuthology.orchestra.run.smithi005.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 973 kB/s | 49 kB 00:00 2024-04-26T04:35:56.117 INFO:teuthology.orchestra.run.smithi005.stdout:(19/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x 8.1 MB/s | 22 MB 00:02 2024-04-26T04:35:56.142 INFO:teuthology.orchestra.run.smithi005.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 269 kB/s | 67 kB 00:00 2024-04-26T04:35:56.167 INFO:teuthology.orchestra.run.smithi005.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 902 kB/s | 309 kB 00:00 2024-04-26T04:35:56.215 INFO:teuthology.orchestra.run.smithi111.stdout:(1/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86_ 1.9 MB/s | 5.0 MB 00:02 2024-04-26T04:35:56.305 INFO:teuthology.orchestra.run.smithi111.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 275 kB/s | 25 kB 00:00 2024-04-26T04:35:56.392 INFO:teuthology.orchestra.run.smithi005.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 5.8 MB/s | 1.6 MB 00:00 2024-04-26T04:35:56.468 INFO:teuthology.orchestra.run.smithi005.stdout:(23/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 9.8 MB/s | 2.9 MB 00:00 2024-04-26T04:35:56.526 INFO:teuthology.orchestra.run.smithi005.stdout:(24/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 8.0 MB/s | 3.1 MB 00:00 2024-04-26T04:35:56.530 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:35:56.530 INFO:teuthology.orchestra.run.smithi005.stdout:Total 16 MB/s | 58 MB 00:03 2024-04-26T04:35:56.539 INFO:teuthology.orchestra.run.smithi111.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 2.7 MB/s | 637 kB 00:00 2024-04-26T04:35:56.593 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:35:56.632 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:35:56.632 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:35:56.655 INFO:teuthology.orchestra.run.smithi111.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 3.9 MB/s | 457 kB 00:00 2024-04-26T04:35:57.090 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:35:57.090 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:35:57.180 INFO:teuthology.orchestra.run.smithi111.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 9.3 MB/s | 4.9 MB 00:00 2024-04-26T04:35:57.322 INFO:teuthology.orchestra.run.smithi111.stdout:(6/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9.x 3.8 MB/s | 14 MB 00:03 2024-04-26T04:35:57.339 INFO:teuthology.orchestra.run.smithi111.stdout:(7/24): python3-ceph-argparse-17.2.7-904.gd8c6b 285 kB/s | 45 kB 00:00 2024-04-26T04:35:57.373 INFO:teuthology.orchestra.run.smithi111.stdout:(8/24): python3-ceph-common-17.2.7-904.gd8c6b0a 2.3 MB/s | 117 kB 00:00 2024-04-26T04:35:57.406 INFO:teuthology.orchestra.run.smithi111.stdout:(9/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 2.3 MB/s | 156 kB 00:00 2024-04-26T04:35:57.457 INFO:teuthology.orchestra.run.smithi111.stdout:(10/24): python3-rados-17.2.7-904.gd8c6b0a3.el9 3.7 MB/s | 320 kB 00:00 2024-04-26T04:35:57.482 INFO:teuthology.orchestra.run.smithi111.stdout:(11/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 3.8 MB/s | 293 kB 00:00 2024-04-26T04:35:57.507 INFO:teuthology.orchestra.run.smithi111.stdout:(12/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.9 MB/s | 100 kB 00:00 2024-04-26T04:35:57.626 INFO:teuthology.orchestra.run.smithi111.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 495 kB/s | 45 kB 00:00 2024-04-26T04:35:57.649 INFO:teuthology.orchestra.run.smithi111.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 958 kB/s | 160 kB 00:00 2024-04-26T04:35:57.708 INFO:teuthology.orchestra.run.smithi111.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 7.7 MB/s | 662 kB 00:00 2024-04-26T04:35:57.733 INFO:teuthology.orchestra.run.smithi111.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-04-26T04:35:57.759 INFO:teuthology.orchestra.run.smithi111.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 841 kB/s | 42 kB 00:00 2024-04-26T04:35:58.184 INFO:teuthology.orchestra.run.smithi111.stdout:(18/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x 4.9 MB/s | 22 MB 00:04 2024-04-26T04:35:58.243 INFO:teuthology.orchestra.run.smithi111.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 100 kB/s | 49 kB 00:00 2024-04-26T04:35:58.358 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:35:58.568 INFO:teuthology.orchestra.run.smithi111.stdout:(20/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 370 kB/s | 309 kB 00:00 2024-04-26T04:35:58.693 INFO:teuthology.orchestra.run.smithi111.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 132 kB/s | 67 kB 00:00 2024-04-26T04:35:58.735 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:35:58.812 INFO:teuthology.orchestra.run.smithi005.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:35:58.827 INFO:teuthology.orchestra.run.smithi111.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 2.7 MB/s | 1.6 MB 00:00 2024-04-26T04:35:58.896 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:35:58.976 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:35:59.013 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:35:59.102 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:35:59.148 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:35:59.176 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:35:59.462 INFO:teuthology.orchestra.run.smithi111.stdout:(23/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 3.4 MB/s | 3.1 MB 00:00 2024-04-26T04:35:59.510 INFO:teuthology.orchestra.run.smithi111.stdout:(24/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 3.6 MB/s | 2.9 MB 00:00 2024-04-26T04:35:59.518 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:35:59.519 INFO:teuthology.orchestra.run.smithi111.stdout:Total 9.1 MB/s | 58 MB 00:06 2024-04-26T04:35:59.583 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:35:59.620 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:35:59.620 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:35:59.634 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:35:59.666 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:35:59.712 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:35:59.749 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:35:59.793 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:35:59.829 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:35:59.893 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:35:59.919 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:35:59.967 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:36:00.007 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:36:00.044 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:36:00.059 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:00.059 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:00.076 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:36:00.385 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:36:00.431 INFO:teuthology.orchestra.run.smithi005.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:36:00.530 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:36:00.607 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:36:00.700 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:36:01.336 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:01.665 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:36:01.772 INFO:teuthology.orchestra.run.smithi111.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:36:01.862 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:36:01.950 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:36:01.990 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:36:02.076 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:36:02.115 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:36:02.144 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:36:02.578 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:36:02.601 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:36:02.655 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:36:02.684 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:36:02.726 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:36:02.764 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:36:02.825 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:36:02.854 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:36:02.910 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:36:02.949 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:36:02.979 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:36:03.011 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:36:03.180 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:36:03.213 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:36:03.302 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:36:03.341 INFO:teuthology.orchestra.run.smithi111.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:36:03.430 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:36:03.504 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:36:03.543 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:36:03.747 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:36:03.785 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:36:03.879 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:36:03.879 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:36:03.879 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:36:03.879 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:03.911 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:36:05.954 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:36:06.073 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:36:06.582 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:36:06.627 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:36:06.722 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:36:06.723 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:36:06.723 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:36:06.723 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:06.783 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /sys 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /proc 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /mnt 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /var/tmp 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /home 2024-04-26T04:36:13.831 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /root 2024-04-26T04:36:13.832 INFO:teuthology.orchestra.run.smithi005.stdout:skipping the directory /tmp 2024-04-26T04:36:13.832 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:15.151 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:36:15.221 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:36:15.222 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:15.222 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:36:15.222 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:36:15.222 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:36:15.222 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:15.223 INFO:teuthology.orchestra.run.smithi005.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:36:15.312 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:36:15.312 INFO:teuthology.orchestra.run.smithi005.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /sys 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /proc 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /mnt 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /var/tmp 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /home 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /root 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout:skipping the directory /tmp 2024-04-26T04:36:16.227 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:36:16.785 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:36:16.786 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:36:16.787 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:36:16.787 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:36:16.787 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:36:17.475 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:36:17.475 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:17.475 INFO:teuthology.orchestra.run.smithi005.stdout:Upgraded: 2024-04-26T04:36:17.475 INFO:teuthology.orchestra.run.smithi005.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.476 INFO:teuthology.orchestra.run.smithi005.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:17.477 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:17.481 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:36:17.559 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:36:17.559 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:17.559 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:36:17.559 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:36:17.559 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:36:17.560 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:17.561 INFO:teuthology.orchestra.run.smithi111.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:36:17.642 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:36:17.643 INFO:teuthology.orchestra.run.smithi111.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:36:17.765 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-test 2024-04-26T04:36:18.307 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:28 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:18.431 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:36:18.432 INFO:teuthology.orchestra.run.smithi005.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout:Install 4 Packages 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:18.433 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 48 M 2024-04-26T04:36:18.434 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 192 M 2024-04-26T04:36:18.434 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:36:18.727 INFO:teuthology.orchestra.run.smithi005.stdout:(1/4): socat-1.7.4.1-5.el9.x86_64.rpm 1.7 MB/s | 305 kB 00:00 2024-04-26T04:36:18.785 INFO:teuthology.orchestra.run.smithi005.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 64 kB 00:00 2024-04-26T04:36:18.953 INFO:teuthology.orchestra.run.smithi005.stdout:(3/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 410 kB/s | 163 kB 00:00 2024-04-26T04:36:19.002 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:36:19.002 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:36:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:36:19.004 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout:Upgraded: 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.741 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:36:19.742 INFO:teuthology.orchestra.run.smithi111.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:19.743 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:20.023 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-test 2024-04-26T04:36:20.553 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:29 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:20.675 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:20.675 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:20.675 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:36:20.675 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:20.675 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout:Install 4 Packages 2024-04-26T04:36:20.676 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:20.677 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 48 M 2024-04-26T04:36:20.677 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 192 M 2024-04-26T04:36:20.677 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:36:20.959 INFO:teuthology.orchestra.run.smithi111.stdout:(1/4): socat-1.7.4.1-5.el9.x86_64.rpm 1.9 MB/s | 305 kB 00:00 2024-04-26T04:36:21.009 INFO:teuthology.orchestra.run.smithi111.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.3 MB/s | 64 kB 00:00 2024-04-26T04:36:21.245 INFO:teuthology.orchestra.run.smithi111.stdout:(3/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 367 kB/s | 163 kB 00:00 2024-04-26T04:36:21.347 INFO:teuthology.orchestra.run.smithi005.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 17 MB/s | 48 MB 00:02 2024-04-26T04:36:21.351 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:21.351 INFO:teuthology.orchestra.run.smithi005.stdout:Total 17 MB/s | 48 MB 00:02 2024-04-26T04:36:21.378 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:36:21.400 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:36:21.400 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:36:21.783 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:36:21.784 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:36:22.326 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:36:22.407 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:36:22.447 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:36:22.475 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:36:23.288 INFO:teuthology.orchestra.run.smithi111.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 19 MB/s | 48 MB 00:02 2024-04-26T04:36:23.291 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:23.291 INFO:teuthology.orchestra.run.smithi111.stdout:Total 19 MB/s | 48 MB 00:02 2024-04-26T04:36:23.317 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:36:23.336 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:36:23.336 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:36:23.699 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:23.699 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:24.207 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:24.289 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:36:24.328 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:36:24.356 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:36:27.104 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:36:27.181 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:36:28.698 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:36:28.770 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:36:29.356 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:36:29.356 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:36:29.356 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:36:29.356 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:36:29.642 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:29.643 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:29.932 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph 2024-04-26T04:36:30.467 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:40 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:30.593 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:36:30.596 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:36:30.597 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:36:30.598 INFO:teuthology.orchestra.run.smithi005.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:36:30.599 INFO:teuthology.orchestra.run.smithi005.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:36:30.599 INFO:teuthology.orchestra.run.smithi005.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:36:30.599 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:36:30.599 INFO:teuthology.orchestra.run.smithi005.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:36:30.599 INFO:teuthology.orchestra.run.smithi005.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:36:30.599 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:36:30.600 INFO:teuthology.orchestra.run.smithi005.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:36:30.601 INFO:teuthology.orchestra.run.smithi005.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:36:30.601 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:30.601 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:36:30.601 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:30.601 INFO:teuthology.orchestra.run.smithi005.stdout:Install 33 Packages 2024-04-26T04:36:30.602 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:30.603 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 27 M 2024-04-26T04:36:30.603 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 96 M 2024-04-26T04:36:30.603 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:36:31.074 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:36:31.075 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:36:31.075 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:36:31.075 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:36:31.320 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:36:31.320 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:31.320 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:36:31.320 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:31.320 INFO:teuthology.orchestra.run.smithi111.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:31.321 INFO:teuthology.orchestra.run.smithi111.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:36:31.321 INFO:teuthology.orchestra.run.smithi111.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:36:31.321 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:31.321 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:31.532 INFO:teuthology.orchestra.run.smithi005.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 38 kB/s | 6.5 kB 00:00 2024-04-26T04:36:31.638 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph 2024-04-26T04:36:31.776 INFO:teuthology.orchestra.run.smithi005.stdout:(2/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 5.1 MB/s | 2.1 MB 00:00 2024-04-26T04:36:32.149 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:41 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:32.149 INFO:teuthology.orchestra.run.smithi005.stdout:(3/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 1.8 MB/s | 1.4 MB 00:00 2024-04-26T04:36:32.217 INFO:teuthology.orchestra.run.smithi005.stdout:(4/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 5.7 MB/s | 3.9 MB 00:00 2024-04-26T04:36:32.275 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:32.278 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:32.278 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:36:32.278 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:36:32.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:36:32.280 INFO:teuthology.orchestra.run.smithi111.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:36:32.281 INFO:teuthology.orchestra.run.smithi111.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout:Install 33 Packages 2024-04-26T04:36:32.282 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:32.283 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 27 M 2024-04-26T04:36:32.283 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 96 M 2024-04-26T04:36:32.284 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:36:32.284 INFO:teuthology.orchestra.run.smithi005.stdout:(5/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 1.8 MB/s | 240 kB 00:00 2024-04-26T04:36:32.358 INFO:teuthology.orchestra.run.smithi005.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.7 MB/s | 253 kB 00:00 2024-04-26T04:36:32.412 INFO:teuthology.orchestra.run.smithi005.stdout:(7/33): python3-cryptography-36.0.1-4.el9.x86_6 9.9 MB/s | 1.2 MB 00:00 2024-04-26T04:36:32.434 INFO:teuthology.orchestra.run.smithi005.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 1.4 MB/s | 106 kB 00:00 2024-04-26T04:36:32.459 INFO:teuthology.orchestra.run.smithi005.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.6 MB/s | 135 kB 00:00 2024-04-26T04:36:32.484 INFO:teuthology.orchestra.run.smithi005.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-04-26T04:36:32.510 INFO:teuthology.orchestra.run.smithi005.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-04-26T04:36:32.535 INFO:teuthology.orchestra.run.smithi005.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 172 kB 00:00 2024-04-26T04:36:32.560 INFO:teuthology.orchestra.run.smithi005.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 694 kB/s | 35 kB 00:00 2024-04-26T04:36:32.585 INFO:teuthology.orchestra.run.smithi005.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-04-26T04:36:32.614 INFO:teuthology.orchestra.run.smithi005.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 831 kB/s | 42 kB 00:00 2024-04-26T04:36:32.778 INFO:teuthology.orchestra.run.smithi005.stdout:(16/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_ 15 MB/s | 15 MB 00:01 2024-04-26T04:36:33.022 INFO:teuthology.orchestra.run.smithi005.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 100 kB/s | 43 kB 00:00 2024-04-26T04:36:33.120 INFO:teuthology.orchestra.run.smithi005.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 107 kB/s | 11 kB 00:00 2024-04-26T04:36:33.162 INFO:teuthology.orchestra.run.smithi005.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 312 kB/s | 172 kB 00:00 2024-04-26T04:36:33.180 INFO:teuthology.orchestra.run.smithi111.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 33 kB/s | 6.5 kB 00:00 2024-04-26T04:36:33.212 INFO:teuthology.orchestra.run.smithi005.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 192 kB/s | 18 kB 00:00 2024-04-26T04:36:33.254 INFO:teuthology.orchestra.run.smithi005.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 252 kB/s | 23 kB 00:00 2024-04-26T04:36:33.296 INFO:teuthology.orchestra.run.smithi005.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 692 kB/s | 358 kB 00:00 2024-04-26T04:36:33.321 INFO:teuthology.orchestra.run.smithi005.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 179 kB/s | 19 kB 00:00 2024-04-26T04:36:33.346 INFO:teuthology.orchestra.run.smithi005.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 213 kB/s | 20 kB 00:00 2024-04-26T04:36:33.347 INFO:teuthology.orchestra.run.smithi111.stdout:(2/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 5.7 MB/s | 2.1 MB 00:00 2024-04-26T04:36:33.397 INFO:teuthology.orchestra.run.smithi005.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 461 kB/s | 46 kB 00:00 2024-04-26T04:36:33.489 INFO:teuthology.orchestra.run.smithi005.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 471 kB/s | 79 kB 00:00 2024-04-26T04:36:33.513 INFO:teuthology.orchestra.run.smithi005.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 140 kB/s | 16 kB 00:00 2024-04-26T04:36:33.539 INFO:teuthology.orchestra.run.smithi005.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-04-26T04:36:33.589 INFO:teuthology.orchestra.run.smithi005.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 894 kB/s | 90 kB 00:00 2024-04-26T04:36:33.615 INFO:teuthology.orchestra.run.smithi005.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 355 kB/s | 36 kB 00:00 2024-04-26T04:36:33.640 INFO:teuthology.orchestra.run.smithi005.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-04-26T04:36:33.698 INFO:teuthology.orchestra.run.smithi005.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 240 kB/s | 20 kB 00:00 2024-04-26T04:36:33.731 INFO:teuthology.orchestra.run.smithi111.stdout:(3/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 7.1 MB/s | 3.9 MB 00:00 2024-04-26T04:36:33.757 INFO:teuthology.orchestra.run.smithi005.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-04-26T04:36:33.761 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:33.761 INFO:teuthology.orchestra.run.smithi005.stdout:Total 8.6 MB/s | 27 MB 00:03 2024-04-26T04:36:33.823 INFO:teuthology.orchestra.run.smithi111.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 2.6 MB/s | 240 kB 00:00 2024-04-26T04:36:33.885 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:36:33.924 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:36:33.925 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:36:33.939 INFO:teuthology.orchestra.run.smithi111.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.1 MB/s | 253 kB 00:00 2024-04-26T04:36:34.015 INFO:teuthology.orchestra.run.smithi111.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 16 MB/s | 1.2 MB 00:00 2024-04-26T04:36:34.173 INFO:teuthology.orchestra.run.smithi111.stdout:(7/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_6 19 MB/s | 15 MB 00:00 2024-04-26T04:36:34.198 INFO:teuthology.orchestra.run.smithi111.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 580 kB/s | 106 kB 00:00 2024-04-26T04:36:34.211 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:36:34.212 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:36:34.224 INFO:teuthology.orchestra.run.smithi111.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.6 MB/s | 135 kB 00:00 2024-04-26T04:36:34.249 INFO:teuthology.orchestra.run.smithi111.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-04-26T04:36:34.274 INFO:teuthology.orchestra.run.smithi111.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-04-26T04:36:34.308 INFO:teuthology.orchestra.run.smithi111.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 2.9 MB/s | 172 kB 00:00 2024-04-26T04:36:34.333 INFO:teuthology.orchestra.run.smithi111.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 594 kB/s | 35 kB 00:00 2024-04-26T04:36:34.359 INFO:teuthology.orchestra.run.smithi111.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-04-26T04:36:34.384 INFO:teuthology.orchestra.run.smithi111.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 833 kB/s | 42 kB 00:00 2024-04-26T04:36:34.418 INFO:teuthology.orchestra.run.smithi111.stdout:(16/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_ 1.0 MB/s | 1.4 MB 00:01 2024-04-26T04:36:34.685 INFO:teuthology.orchestra.run.smithi111.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 133 kB/s | 43 kB 00:00 2024-04-26T04:36:34.768 INFO:teuthology.orchestra.run.smithi111.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 447 kB/s | 172 kB 00:00 2024-04-26T04:36:34.794 INFO:teuthology.orchestra.run.smithi111.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 98 kB/s | 11 kB 00:00 2024-04-26T04:36:34.852 INFO:teuthology.orchestra.run.smithi111.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 211 kB/s | 18 kB 00:00 2024-04-26T04:36:34.877 INFO:teuthology.orchestra.run.smithi111.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 277 kB/s | 23 kB 00:00 2024-04-26T04:36:34.903 INFO:teuthology.orchestra.run.smithi111.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 740 kB/s | 358 kB 00:00 2024-04-26T04:36:34.936 INFO:teuthology.orchestra.run.smithi111.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 232 kB/s | 19 kB 00:00 2024-04-26T04:36:34.961 INFO:teuthology.orchestra.run.smithi111.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 235 kB/s | 20 kB 00:00 2024-04-26T04:36:34.987 INFO:teuthology.orchestra.run.smithi111.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 552 kB/s | 46 kB 00:00 2024-04-26T04:36:35.062 INFO:teuthology.orchestra.run.smithi111.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 219 kB/s | 16 kB 00:00 2024-04-26T04:36:35.087 INFO:teuthology.orchestra.run.smithi111.stdout:(27/33): python3-more-itertools-8.12.0-2.el9.no 523 kB/s | 79 kB 00:00 2024-04-26T04:36:35.112 INFO:teuthology.orchestra.run.smithi111.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-04-26T04:36:35.163 INFO:teuthology.orchestra.run.smithi111.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 895 kB/s | 90 kB 00:00 2024-04-26T04:36:35.188 INFO:teuthology.orchestra.run.smithi111.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 355 kB/s | 36 kB 00:00 2024-04-26T04:36:35.213 INFO:teuthology.orchestra.run.smithi111.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-04-26T04:36:35.264 INFO:teuthology.orchestra.run.smithi111.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 265 kB/s | 20 kB 00:00 2024-04-26T04:36:35.355 INFO:teuthology.orchestra.run.smithi111.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.2 MB/s | 427 kB 00:00 2024-04-26T04:36:35.361 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:35.361 INFO:teuthology.orchestra.run.smithi111.stdout:Total 8.8 MB/s | 27 MB 00:03 2024-04-26T04:36:35.475 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:36:35.507 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:36:35.511 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:36:35.511 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:36:35.562 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:36:35.595 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:36:35.627 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:36:35.723 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:36:35.781 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:35.782 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:35.809 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:36:35.855 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:36:35.890 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:36:35.920 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:36:35.955 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:36:35.988 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:36:36.029 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:36:36.072 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:36:36.106 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:36:36.145 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:36:36.221 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:36:36.374 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:36:36.455 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:36:36.507 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:36:36.545 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:36:36.603 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:36:36.666 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:36:36.863 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:36:36.911 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:36.923 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:36:36.966 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:36:36.999 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:36:37.004 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:36:37.032 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:36:37.103 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:36:37.125 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:36:37.138 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:36:37.211 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:36:37.252 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:36:37.262 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:36:37.286 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:36:37.318 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:36:37.350 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:36:37.383 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:36:37.416 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:36:37.424 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:36:37.449 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:36:37.459 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:36:37.476 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:36:37.476 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:37.476 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:36:37.476 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:36:37.477 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:36:37.477 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:37.502 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:36:37.540 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:36:37.614 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:36:37.753 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:36:37.857 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:36:37.910 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:36:37.948 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:36:37.997 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:36:38.053 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:36:38.241 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:36:38.294 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:36:38.373 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:36:38.461 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:36:38.510 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:36:38.628 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:36:38.773 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:36:38.803 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:36:38.827 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:36:38.828 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:38.828 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:36:38.828 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:36:38.828 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:36:38.828 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:38.866 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:36:38.895 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:36:38.895 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:38.895 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:36:38.895 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:36:38.895 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:36:38.895 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:39.284 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:36:39.308 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:36:39.308 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:39.308 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:36:39.308 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:36:39.308 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:36:39.308 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:39.517 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:36:39.542 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:36:39.542 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:39.542 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:36:39.542 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:36:39.542 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:36:39.543 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:39.607 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:36:40.197 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:36:40.225 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:36:40.225 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:40.225 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:36:40.225 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:36:40.225 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:36:40.225 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:40.606 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:36:40.629 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:36:40.630 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:40.630 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:36:40.630 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:36:40.630 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:36:40.630 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:40.831 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:36:40.855 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:36:40.855 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:40.855 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:36:40.855 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:36:40.855 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:36:40.855 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:40.919 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:36:41.221 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:36:41.222 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:36:41.223 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:36:42.110 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:36:42.111 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:36:42.112 INFO:teuthology.orchestra.run.smithi005.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:42.113 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:42.480 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-base 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:36:42.524 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:36:42.525 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:36:42.526 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:36:43.013 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:53 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:43.094 INFO:teuthology.orchestra.run.smithi005.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:36:43.137 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:43.138 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:36:43.138 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:43.210 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install cephadm 2024-04-26T04:36:43.307 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:36:43.308 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:36:43.309 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:43.310 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:43.651 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-base 2024-04-26T04:36:43.737 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:53 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:43.861 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 82 k 2024-04-26T04:36:43.862 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 372 k 2024-04-26T04:36:43.863 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:36:44.082 INFO:teuthology.orchestra.run.smithi005.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 374 kB/s | 82 kB 00:00 2024-04-26T04:36:44.083 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:44.083 INFO:teuthology.orchestra.run.smithi005.stdout:Total 371 kB/s | 82 kB 00:00 2024-04-26T04:36:44.084 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:36:44.089 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:36:44.089 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:36:44.091 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:36:44.091 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:36:44.159 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:53 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:44.160 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:36:44.237 INFO:teuthology.orchestra.run.smithi111.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:36:44.277 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:44.278 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:36:44.278 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:44.336 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install cephadm 2024-04-26T04:36:44.573 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:44.654 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:44.839 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:53 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:44.946 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:44.958 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:44.959 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:44.959 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:36:44.959 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 82 k 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 372 k 2024-04-26T04:36:44.960 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:36:45.185 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:45.185 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:45.185 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:36:45.185 INFO:teuthology.orchestra.run.smithi005.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:45.185 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:45.186 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:45.284 INFO:teuthology.orchestra.run.smithi111.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 256 kB/s | 82 kB 00:00 2024-04-26T04:36:45.284 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:45.284 INFO:teuthology.orchestra.run.smithi111.stdout:Total 254 kB/s | 82 kB 00:00 2024-04-26T04:36:45.284 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:36:45.288 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:36:45.288 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:36:45.289 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:45.289 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:45.361 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:45.420 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:36:45.754 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:45.851 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:45.954 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:55 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:46.080 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:46.081 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:46.081 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repo Size 2024-04-26T04:36:46.081 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:36:46.082 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:46.083 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 137 k 2024-04-26T04:36:46.083 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 379 k 2024-04-26T04:36:46.083 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:36:46.159 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:46.307 INFO:teuthology.orchestra.run.smithi005.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 614 kB/s | 137 kB 00:00 2024-04-26T04:36:46.307 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:46.307 INFO:teuthology.orchestra.run.smithi005.stdout:Total 611 kB/s | 137 kB 00:00 2024-04-26T04:36:46.307 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:36:46.314 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:36:46.314 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:36:46.373 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:36:46.373 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:36:46.410 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:36:46.410 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:46.411 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:36:46.411 INFO:teuthology.orchestra.run.smithi111.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:46.411 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:46.411 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:46.518 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:36:46.564 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:36:46.590 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:36:46.590 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:46.591 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:36:46.591 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:46.627 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:36:47.137 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:56 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:47.256 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repo Size 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:36:47.257 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:47.258 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:36:47.258 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:47.258 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 137 k 2024-04-26T04:36:47.258 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 379 k 2024-04-26T04:36:47.258 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:36:47.502 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:36:47.502 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:47.502 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:36:47.502 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:47.502 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:47.502 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:47.596 INFO:teuthology.orchestra.run.smithi111.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 407 kB/s | 137 kB 00:00 2024-04-26T04:36:47.596 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:47.597 INFO:teuthology.orchestra.run.smithi111.stdout:Total 405 kB/s | 137 kB 00:00 2024-04-26T04:36:47.597 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:36:47.605 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:36:47.605 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:36:47.660 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:47.660 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:47.759 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-mgr 2024-04-26T04:36:47.815 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:47.896 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:36:47.920 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:36:47.920 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:36:47.920 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:36:47.921 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:48.287 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:58 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:48.368 INFO:teuthology.orchestra.run.smithi005.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:36:48.411 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:48.412 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:36:48.412 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:48.489 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:36:48.759 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:36:48.759 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:48.759 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:36:48.759 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:36:48.759 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:48.759 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:48.940 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-mgr 2024-04-26T04:36:49.029 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:49.154 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:36:49.155 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout:Installing weak dependencies: 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout:Install 7 Packages 2024-04-26T04:36:49.156 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:49.157 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 3.7 M 2024-04-26T04:36:49.157 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 80 M 2024-04-26T04:36:49.157 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:36:49.449 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:58 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:49.509 INFO:teuthology.orchestra.run.smithi005.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 74 kB/s | 15 kB 00:00 2024-04-26T04:36:49.528 INFO:teuthology.orchestra.run.smithi111.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:36:49.569 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:49.570 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:36:49.575 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:49.626 INFO:teuthology.orchestra.run.smithi005.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 78 kB/s | 24 kB 00:00 2024-04-26T04:36:49.639 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:36:49.751 INFO:teuthology.orchestra.run.smithi005.stdout:(3/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 7.6 MB/s | 3.4 MB 00:00 2024-04-26T04:36:49.768 INFO:teuthology.orchestra.run.smithi005.stdout:(4/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 35 kB/s | 9.0 kB 00:00 2024-04-26T04:36:49.927 INFO:teuthology.orchestra.run.smithi005.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 175 kB/s | 31 kB 00:00 2024-04-26T04:36:50.002 INFO:teuthology.orchestra.run.smithi005.stdout:(6/7): python3-jwt-2.4.0-1.el9.noarch.rpm 109 kB/s | 41 kB 00:00 2024-04-26T04:36:50.158 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:50.160 INFO:teuthology.orchestra.run.smithi005.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 481 kB/s | 188 kB 00:00 2024-04-26T04:36:50.163 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:50.163 INFO:teuthology.orchestra.run.smithi005.stdout:Total 3.6 MB/s | 3.7 MB 00:01 2024-04-26T04:36:50.163 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:36:50.174 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:36:50.174 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:36:50.246 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:36:50.246 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:36:50.277 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:50.278 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:50.278 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:36:50.278 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout:Installing weak dependencies: 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:36:50.279 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:36:50.280 INFO:teuthology.orchestra.run.smithi111.stdout:Install 7 Packages 2024-04-26T04:36:50.280 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:50.280 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 3.7 M 2024-04-26T04:36:50.280 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 80 M 2024-04-26T04:36:50.280 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:36:50.553 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:36:50.620 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:36:50.655 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:36:50.675 INFO:teuthology.orchestra.run.smithi111.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 70 kB/s | 15 kB 00:00 2024-04-26T04:36:50.711 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:36:50.749 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:36:50.778 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:36:50.808 INFO:teuthology.orchestra.run.smithi111.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 71 kB/s | 24 kB 00:00 2024-04-26T04:36:50.850 INFO:teuthology.orchestra.run.smithi111.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 52 kB/s | 9.0 kB 00:00 2024-04-26T04:36:51.026 INFO:teuthology.orchestra.run.smithi111.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 176 kB/s | 31 kB 00:00 2024-04-26T04:36:51.118 INFO:teuthology.orchestra.run.smithi111.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 132 kB/s | 41 kB 00:00 2024-04-26T04:36:51.268 INFO:teuthology.orchestra.run.smithi111.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 779 kB/s | 188 kB 00:00 2024-04-26T04:36:51.793 INFO:teuthology.orchestra.run.smithi111.stdout:(7/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 2.5 MB/s | 3.4 MB 00:01 2024-04-26T04:36:51.795 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:51.796 INFO:teuthology.orchestra.run.smithi111.stdout:Total 2.4 MB/s | 3.7 MB 00:01 2024-04-26T04:36:51.796 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:36:51.805 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:36:51.805 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:36:51.837 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:36:51.873 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:36:51.874 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:51.874 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:52.181 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:52.245 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:36:52.279 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:36:52.325 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:36:52.365 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:36:52.394 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:36:52.922 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:36:52.923 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:36:52.923 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:36:52.923 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:36:52.923 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:36:52.923 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:36:52.923 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:36:53.200 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:36:53.200 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:53.200 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:36:53.200 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:53.200 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:53.200 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:53.201 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:36:53.201 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:36:53.201 INFO:teuthology.orchestra.run.smithi005.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:36:53.201 INFO:teuthology.orchestra.run.smithi005.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:36:53.201 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:53.201 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:36:53.394 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:36:53.431 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:36:53.449 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:36:53.984 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:03 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:36:54.111 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout:====================================================================================== 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout:====================================================================================== 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:36:54.113 INFO:teuthology.orchestra.run.smithi005.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout:====================================================================================== 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout:Install 12 Packages 2024-04-26T04:36:54.114 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:36:54.115 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 38 M 2024-04-26T04:36:54.115 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 209 M 2024-04-26T04:36:54.115 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:36:54.480 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:36:54.643 INFO:teuthology.orchestra.run.smithi005.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.1 MB/s | 195 kB 00:00 2024-04-26T04:36:54.668 INFO:teuthology.orchestra.run.smithi005.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.8 MB/s | 803 kB 00:00 2024-04-26T04:36:54.693 INFO:teuthology.orchestra.run.smithi005.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 664 kB/s | 33 kB 00:00 2024-04-26T04:36:54.725 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:36:54.725 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:54.725 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:36:54.725 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:54.726 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:36:54.735 INFO:teuthology.orchestra.run.smithi005.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 448 kB/s | 18 kB 00:00 2024-04-26T04:36:54.786 INFO:teuthology.orchestra.run.smithi005.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 26 MB/s | 3.0 MB 00:00 2024-04-26T04:36:54.811 INFO:teuthology.orchestra.run.smithi005.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 480 kB/s | 35 kB 00:00 2024-04-26T04:36:54.861 INFO:teuthology.orchestra.run.smithi005.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 4.8 MB/s | 244 kB 00:00 2024-04-26T04:36:54.913 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:36:54.945 INFO:teuthology.orchestra.run.smithi005.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 30 MB/s | 4.7 MB 00:00 2024-04-26T04:36:55.029 INFO:teuthology.orchestra.run.smithi005.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 31 MB/s | 5.1 MB 00:00 2024-04-26T04:36:55.046 INFO:teuthology.orchestra.run.smithi005.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.5 MB/s | 250 kB 00:00 2024-04-26T04:36:55.137 INFO:teuthology.orchestra.run.smithi005.stdout:(11/12): ceph-mgr-diskprediction-local-17.2.7-9 11 MB/s | 7.4 MB 00:00 2024-04-26T04:36:55.420 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:04 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:36:55.429 INFO:teuthology.orchestra.run.smithi005.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 41 MB/s | 16 MB 00:00 2024-04-26T04:36:55.432 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:55.432 INFO:teuthology.orchestra.run.smithi005.stdout:Total 29 MB/s | 38 MB 00:01 2024-04-26T04:36:55.541 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout:====================================================================================== 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout:====================================================================================== 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:36:55.543 INFO:teuthology.orchestra.run.smithi111.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout:====================================================================================== 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout:Install 12 Packages 2024-04-26T04:36:55.544 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:36:55.545 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 38 M 2024-04-26T04:36:55.545 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 209 M 2024-04-26T04:36:55.545 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:36:55.792 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:36:55.820 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:36:55.821 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:36:56.253 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:36:56.253 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:36:56.259 INFO:teuthology.orchestra.run.smithi111.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.2 MB/s | 195 kB 00:00 2024-04-26T04:36:56.293 INFO:teuthology.orchestra.run.smithi111.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 4.1 MB/s | 803 kB 00:00 2024-04-26T04:36:56.318 INFO:teuthology.orchestra.run.smithi111.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 566 kB/s | 33 kB 00:00 2024-04-26T04:36:56.360 INFO:teuthology.orchestra.run.smithi111.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 447 kB/s | 18 kB 00:00 2024-04-26T04:36:56.419 INFO:teuthology.orchestra.run.smithi111.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 24 MB/s | 3.0 MB 00:00 2024-04-26T04:36:56.444 INFO:teuthology.orchestra.run.smithi111.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 429 kB/s | 35 kB 00:00 2024-04-26T04:36:56.486 INFO:teuthology.orchestra.run.smithi111.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 6.0 MB/s | 244 kB 00:00 2024-04-26T04:36:56.561 INFO:teuthology.orchestra.run.smithi111.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 33 MB/s | 4.7 MB 00:00 2024-04-26T04:36:56.662 INFO:teuthology.orchestra.run.smithi111.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 29 MB/s | 5.1 MB 00:00 2024-04-26T04:36:56.687 INFO:teuthology.orchestra.run.smithi111.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.0 MB/s | 250 kB 00:00 2024-04-26T04:36:56.971 INFO:teuthology.orchestra.run.smithi111.stdout:(11/12): ceph-mgr-diskprediction-local-17.2.7-9 8.5 MB/s | 7.4 MB 00:00 2024-04-26T04:36:57.036 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:36:57.101 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:36:57.121 INFO:teuthology.orchestra.run.smithi111.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-04-26T04:36:57.123 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:36:57.124 INFO:teuthology.orchestra.run.smithi111.stdout:Total 24 MB/s | 38 MB 00:01 2024-04-26T04:36:57.145 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:36:57.249 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:36:57.287 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:36:57.472 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:36:57.472 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:36:57.500 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:36:57.500 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:36:57.503 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:36:57.582 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:36:57.905 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:36:57.905 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:36:58.269 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:36:58.389 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:36:58.656 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:36:58.725 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:36:58.768 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:36:58.867 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:36:58.903 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:36:59.080 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:36:59.110 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:36:59.187 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:36:59.696 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:36:59.829 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:36:59.932 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:37:00.781 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:37:00.810 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:37:01.151 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:37:02.176 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:37:02.202 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:37:03.479 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:37:03.480 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:37:03.480 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:37:03.481 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:37:03.884 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:37:03.884 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:03.884 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:03.884 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:37:03.885 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:03.886 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:04.271 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:37:04.696 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:37:04.697 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:37:04.697 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:37:04.697 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:37:04.697 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:37:04.697 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:37:04.811 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:14 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:04.939 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:37:04.940 INFO:teuthology.orchestra.run.smithi005.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:37:04.941 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:04.942 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:04.942 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:04.942 INFO:teuthology.orchestra.run.smithi005.stdout:Install 13 Packages 2024-04-26T04:37:04.942 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:04.942 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 2.2 M 2024-04-26T04:37:04.942 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 26 M 2024-04-26T04:37:04.943 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:05.098 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:37:05.098 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:05.098 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:05.098 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:37:05.099 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:05.100 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:05.401 INFO:teuthology.orchestra.run.smithi005.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 109 kB/s | 19 kB 00:00 2024-04-26T04:37:05.426 INFO:teuthology.orchestra.run.smithi005.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 127 kB/s | 26 kB 00:00 2024-04-26T04:37:05.444 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:37:05.577 INFO:teuthology.orchestra.run.smithi005.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.2 MB/s | 222 kB 00:00 2024-04-26T04:37:05.602 INFO:teuthology.orchestra.run.smithi005.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 911 kB/s | 159 kB 00:00 2024-04-26T04:37:05.669 INFO:teuthology.orchestra.run.smithi005.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.0 MB/s | 279 kB 00:00 2024-04-26T04:37:05.694 INFO:teuthology.orchestra.run.smithi005.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 583 kB/s | 54 kB 00:00 2024-04-26T04:37:05.778 INFO:teuthology.orchestra.run.smithi005.stdout:(7/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 86 kB/s | 47 kB 00:00 2024-04-26T04:37:05.861 INFO:teuthology.orchestra.run.smithi005.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 90 kB/s | 15 kB 00:00 2024-04-26T04:37:05.887 INFO:teuthology.orchestra.run.smithi005.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 148 kB/s | 32 kB 00:00 2024-04-26T04:37:05.961 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:14 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:06.021 INFO:teuthology.orchestra.run.smithi005.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 442 kB/s | 59 kB 00:00 2024-04-26T04:37:06.080 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:06.081 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:06.081 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:37:06.081 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:06.081 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:06.081 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:37:06.081 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:37:06.082 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:06.083 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:06.083 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:06.083 INFO:teuthology.orchestra.run.smithi111.stdout:Install 13 Packages 2024-04-26T04:37:06.083 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:06.084 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 2.2 M 2024-04-26T04:37:06.084 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 26 M 2024-04-26T04:37:06.084 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:06.129 INFO:teuthology.orchestra.run.smithi005.stdout:(11/13): python3-google-auth-2.29.0-1.el9.noarc 586 kB/s | 206 kB 00:00 2024-04-26T04:37:06.154 INFO:teuthology.orchestra.run.smithi005.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 672 kB/s | 90 kB 00:00 2024-04-26T04:37:06.279 INFO:teuthology.orchestra.run.smithi005.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 2.4 MB/s | 1.0 MB 00:00 2024-04-26T04:37:06.282 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:06.282 INFO:teuthology.orchestra.run.smithi005.stdout:Total 1.6 MB/s | 2.2 MB 00:01 2024-04-26T04:37:06.346 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:06.361 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:06.361 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:06.453 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:06.453 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:06.676 INFO:teuthology.orchestra.run.smithi111.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 153 kB/s | 26 kB 00:00 2024-04-26T04:37:06.702 INFO:teuthology.orchestra.run.smithi111.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 99 kB/s | 19 kB 00:00 2024-04-26T04:37:06.818 INFO:teuthology.orchestra.run.smithi111.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-04-26T04:37:06.844 INFO:teuthology.orchestra.run.smithi111.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2024-04-26T04:37:06.877 INFO:teuthology.orchestra.run.smithi111.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.7 MB/s | 279 kB 00:00 2024-04-26T04:37:06.903 INFO:teuthology.orchestra.run.smithi111.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 917 kB/s | 54 kB 00:00 2024-04-26T04:37:06.936 INFO:teuthology.orchestra.run.smithi111.stdout:(7/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 111 kB/s | 47 kB 00:00 2024-04-26T04:37:07.037 INFO:teuthology.orchestra.run.smithi111.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 112 kB/s | 15 kB 00:00 2024-04-26T04:37:07.055 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:07.062 INFO:teuthology.orchestra.run.smithi111.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 175 kB/s | 32 kB 00:00 2024-04-26T04:37:07.170 INFO:teuthology.orchestra.run.smithi111.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 545 kB/s | 59 kB 00:00 2024-04-26T04:37:07.196 INFO:teuthology.orchestra.run.smithi111.stdout:(11/13): python3-google-auth-2.29.0-1.el9.noarc 793 kB/s | 206 kB 00:00 2024-04-26T04:37:07.236 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:37:07.254 INFO:teuthology.orchestra.run.smithi111.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.0 MB/s | 90 kB 00:00 2024-04-26T04:37:07.292 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:37:07.304 INFO:teuthology.orchestra.run.smithi111.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 3.8 MB/s | 1.0 MB 00:00 2024-04-26T04:37:07.306 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:07.307 INFO:teuthology.orchestra.run.smithi111.stdout:Total 1.8 MB/s | 2.2 MB 00:01 2024-04-26T04:37:07.348 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:37:07.368 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:07.383 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:07.383 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:07.383 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:37:07.425 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:37:07.464 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:07.464 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:07.523 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:37:07.635 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:37:07.686 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:37:08.040 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:08.218 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:37:08.273 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:37:08.328 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:37:08.366 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:37:08.407 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:37:08.452 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:37:08.501 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:37:08.506 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:37:08.540 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:37:08.586 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:37:08.609 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:37:08.610 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:37:08.658 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:37:09.331 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:37:09.332 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:37:09.368 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:37:09.422 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:37:09.456 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:37:09.501 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:37:09.526 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:37:09.657 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:37:09.657 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:09.657 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:09.657 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:37:09.657 INFO:teuthology.orchestra.run.smithi005.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:37:09.658 INFO:teuthology.orchestra.run.smithi005.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:37:09.659 INFO:teuthology.orchestra.run.smithi005.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:37:09.659 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:09.659 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:09.892 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:37:10.335 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:37:10.336 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:37:10.336 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:37:10.428 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:20 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:10.552 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:10.552 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:10.553 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:10.554 INFO:teuthology.orchestra.run.smithi005.stdout:Install 6 Packages 2024-04-26T04:37:10.554 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:10.554 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 7.0 M 2024-04-26T04:37:10.554 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 33 M 2024-04-26T04:37:10.554 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:37:10.642 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:10.643 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:10.862 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:37:11.314 INFO:teuthology.orchestra.run.smithi005.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 802 kB/s | 248 kB 00:00 2024-04-26T04:37:11.379 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:20 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:11.498 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:37:11.499 INFO:teuthology.orchestra.run.smithi111.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout:Install 6 Packages 2024-04-26T04:37:11.500 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:11.501 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 7.0 M 2024-04-26T04:37:11.501 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 33 M 2024-04-26T04:37:11.501 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:11.606 INFO:teuthology.orchestra.run.smithi005.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 9.9 MB/s | 6.0 MB 00:00 2024-04-26T04:37:11.790 INFO:teuthology.orchestra.run.smithi005.stdout:(3/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 157 kB/s | 123 kB 00:00 2024-04-26T04:37:12.007 INFO:teuthology.orchestra.run.smithi005.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 793 kB/s | 548 kB 00:00 2024-04-26T04:37:12.057 INFO:teuthology.orchestra.run.smithi005.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 128 kB/s | 58 kB 00:00 2024-04-26T04:37:12.215 INFO:teuthology.orchestra.run.smithi005.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 120 kB/s | 51 kB 00:00 2024-04-26T04:37:12.220 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:12.220 INFO:teuthology.orchestra.run.smithi005.stdout:Total 4.2 MB/s | 7.0 MB 00:01 2024-04-26T04:37:12.293 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:12.306 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:12.306 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:12.409 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:12.409 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:12.424 INFO:teuthology.orchestra.run.smithi111.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.3 MB/s | 248 kB 00:00 2024-04-26T04:37:12.633 INFO:teuthology.orchestra.run.smithi111.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 6.0 MB 00:00 2024-04-26T04:37:12.719 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:12.766 INFO:teuthology.orchestra.run.smithi111.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 434 kB/s | 58 kB 00:00 2024-04-26T04:37:12.833 INFO:teuthology.orchestra.run.smithi111.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 765 kB/s | 51 kB 00:00 2024-04-26T04:37:12.858 INFO:teuthology.orchestra.run.smithi111.stdout:(5/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 197 kB/s | 123 kB 00:00 2024-04-26T04:37:12.898 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:37:12.952 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:37:13.350 INFO:teuthology.orchestra.run.smithi111.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 593 kB/s | 548 kB 00:00 2024-04-26T04:37:13.352 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:13.353 INFO:teuthology.orchestra.run.smithi111.stdout:Total 3.8 MB/s | 7.0 MB 00:01 2024-04-26T04:37:13.424 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:13.436 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:13.436 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:13.508 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:37:13.532 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:13.533 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:13.607 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:37:13.669 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:37:13.696 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:37:13.829 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:13.998 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:37:14.052 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:37:14.431 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:37:14.431 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:37:14.431 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:37:14.431 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:37:14.431 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:37:14.431 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:37:14.569 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:37:14.655 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:37:14.689 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:37:14.689 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:14.689 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:14.689 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:37:14.689 INFO:teuthology.orchestra.run.smithi005.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:37:14.689 INFO:teuthology.orchestra.run.smithi005.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:37:14.690 INFO:teuthology.orchestra.run.smithi005.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:37:14.690 INFO:teuthology.orchestra.run.smithi005.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:37:14.690 INFO:teuthology.orchestra.run.smithi005.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:37:14.690 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:14.690 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:14.709 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:37:14.739 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:37:14.931 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install ceph-fuse 2024-04-26T04:37:15.473 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:25 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:15.497 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:37:15.497 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:37:15.497 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:37:15.497 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:37:15.497 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:37:15.497 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:37:15.598 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repository Size 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout:Installing dependencies: 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:15.599 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:15.600 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:15.600 INFO:teuthology.orchestra.run.smithi005.stdout:Install 2 Packages 2024-04-26T04:37:15.600 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:15.600 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 892 k 2024-04-26T04:37:15.601 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 2.6 M 2024-04-26T04:37:15.601 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:15.738 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:37:15.738 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:15.738 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:15.738 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:37:15.738 INFO:teuthology.orchestra.run.smithi111.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:37:15.739 INFO:teuthology.orchestra.run.smithi111.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:37:15.739 INFO:teuthology.orchestra.run.smithi111.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:37:15.739 INFO:teuthology.orchestra.run.smithi111.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:37:15.739 INFO:teuthology.orchestra.run.smithi111.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:37:15.739 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:15.739 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:15.901 INFO:teuthology.orchestra.run.smithi005.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 535 kB/s | 80 kB 00:00 2024-04-26T04:37:15.989 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install ceph-fuse 2024-04-26T04:37:16.505 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:25 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:16.625 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repository Size 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout:Installing dependencies: 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:16.626 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:16.627 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:16.627 INFO:teuthology.orchestra.run.smithi111.stdout:Install 2 Packages 2024-04-26T04:37:16.627 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:16.627 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 892 k 2024-04-26T04:37:16.627 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 2.6 M 2024-04-26T04:37:16.627 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:16.859 INFO:teuthology.orchestra.run.smithi005.stdout:(2/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 733 kB/s | 812 kB 00:01 2024-04-26T04:37:16.861 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:16.861 INFO:teuthology.orchestra.run.smithi005.stdout:Total 707 kB/s | 892 kB 00:01 2024-04-26T04:37:16.875 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:16.901 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:16.901 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:16.987 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:16.987 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:16.994 INFO:teuthology.orchestra.run.smithi111.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 437 kB/s | 80 kB 00:00 2024-04-26T04:37:17.177 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:17.300 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:37:17.375 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:37:17.577 INFO:teuthology.orchestra.run.smithi111.stdout:(2/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 1.0 MB/s | 812 kB 00:00 2024-04-26T04:37:17.579 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:17.579 INFO:teuthology.orchestra.run.smithi111.stdout:Total 937 kB/s | 892 kB 00:00 2024-04-26T04:37:17.592 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:17.614 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:17.615 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:17.693 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:17.693 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:17.874 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:18.007 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:37:18.062 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:37:18.063 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:37:18.075 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:37:18.330 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:37:18.330 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:18.330 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:18.330 INFO:teuthology.orchestra.run.smithi005.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:37:18.330 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:18.331 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:18.573 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install librados-devel 2024-04-26T04:37:18.764 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:37:18.764 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:37:19.002 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:37:19.002 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:19.002 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:37:19.003 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:19.003 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:19.107 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:29 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:19.232 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:19.232 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:19.232 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repo Size 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 126 k 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 448 k 2024-04-26T04:37:19.233 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:19.314 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install librados-devel 2024-04-26T04:37:19.827 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:28 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:19.947 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repo Size 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 126 k 2024-04-26T04:37:19.948 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 448 k 2024-04-26T04:37:19.949 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:20.047 INFO:teuthology.orchestra.run.smithi005.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 155 kB/s | 126 kB 00:00 2024-04-26T04:37:20.048 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:20.048 INFO:teuthology.orchestra.run.smithi005.stdout:Total 155 kB/s | 126 kB 00:00 2024-04-26T04:37:20.048 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:20.055 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:20.056 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:20.127 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:20.128 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:20.297 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:20.445 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:20.716 INFO:teuthology.orchestra.run.smithi111.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 165 kB/s | 126 kB 00:00 2024-04-26T04:37:20.716 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:20.717 INFO:teuthology.orchestra.run.smithi111.stdout:Total 164 kB/s | 126 kB 00:00 2024-04-26T04:37:20.717 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:20.722 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:20.722 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:20.787 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:20.787 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:20.831 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:20.942 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:21.070 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:21.079 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:21.080 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:21.080 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:21.080 INFO:teuthology.orchestra.run.smithi005.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:21.080 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:21.080 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:21.324 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install libcephfs2 2024-04-26T04:37:21.452 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:21.679 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:21.679 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:21.679 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:21.679 INFO:teuthology.orchestra.run.smithi111.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:21.679 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:21.679 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:21.872 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:31 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:21.943 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install libcephfs2 2024-04-26T04:37:21.955 INFO:teuthology.orchestra.run.smithi005.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:21.998 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:21.999 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:21.999 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:22.072 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install libcephfs-devel 2024-04-26T04:37:22.459 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:31 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:22.539 INFO:teuthology.orchestra.run.smithi111.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:22.580 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:22.581 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:22.582 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:22.599 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:22.652 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install libcephfs-devel 2024-04-26T04:37:22.723 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:22.726 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:22.726 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repo Size 2024-04-26T04:37:22.726 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 25 k 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 110 k 2024-04-26T04:37:22.727 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:23.052 INFO:teuthology.orchestra.run.smithi005.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 76 kB/s | 25 kB 00:00 2024-04-26T04:37:23.053 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:23.053 INFO:teuthology.orchestra.run.smithi005.stdout:Total 76 kB/s | 25 kB 00:00 2024-04-26T04:37:23.053 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:23.057 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:23.057 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:23.058 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:23.059 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:23.145 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:23.153 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:23.242 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:23.272 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repo Size 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:23.273 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:37:23.274 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:23.274 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 25 k 2024-04-26T04:37:23.274 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 110 k 2024-04-26T04:37:23.274 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:23.588 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:23.636 INFO:teuthology.orchestra.run.smithi111.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 69 kB/s | 25 kB 00:00 2024-04-26T04:37:23.637 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:23.637 INFO:teuthology.orchestra.run.smithi111.stdout:Total 68 kB/s | 25 kB 00:00 2024-04-26T04:37:23.637 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:23.640 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:23.640 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:23.642 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:23.642 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:23.728 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:23.821 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:23.821 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:23.822 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:23.822 INFO:teuthology.orchestra.run.smithi005.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:23.822 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:23.822 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:23.823 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:23.995 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install librados2 2024-04-26T04:37:24.181 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:24.403 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:24.403 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:24.403 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:24.403 INFO:teuthology.orchestra.run.smithi111.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:24.403 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:24.403 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:24.527 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:24.593 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install librados2 2024-04-26T04:37:24.608 INFO:teuthology.orchestra.run.smithi005.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:24.651 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:24.652 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:24.652 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:24.722 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install librbd1 2024-04-26T04:37:25.103 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:25.182 INFO:teuthology.orchestra.run.smithi111.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:25.221 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:25.223 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:25.223 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:25.256 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:25.300 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install librbd1 2024-04-26T04:37:25.336 INFO:teuthology.orchestra.run.smithi005.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:25.378 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:25.379 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:25.379 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:25.451 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install python3-rados 2024-04-26T04:37:25.806 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:25.883 INFO:teuthology.orchestra.run.smithi111.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:25.924 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:25.925 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:25.925 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:25.983 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:25.994 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install python3-rados 2024-04-26T04:37:26.064 INFO:teuthology.orchestra.run.smithi005.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:26.107 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:26.108 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:26.108 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:26.183 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install python3-rgw 2024-04-26T04:37:26.506 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:26.585 INFO:teuthology.orchestra.run.smithi111.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:26.626 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:26.627 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:26.627 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:26.699 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install python3-rgw 2024-04-26T04:37:26.712 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:26.793 INFO:teuthology.orchestra.run.smithi005.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:26.836 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:26.837 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:26.837 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:26.905 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install python3-cephfs 2024-04-26T04:37:27.205 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:27.284 INFO:teuthology.orchestra.run.smithi111.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:27.325 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:27.326 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:27.326 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:27.391 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install python3-cephfs 2024-04-26T04:37:27.437 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:27.518 INFO:teuthology.orchestra.run.smithi005.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:27.561 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:27.562 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:27.562 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:27.643 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install python3-rbd 2024-04-26T04:37:27.899 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:27.977 INFO:teuthology.orchestra.run.smithi111.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:28.018 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:28.019 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:28.019 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:28.091 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install python3-rbd 2024-04-26T04:37:28.181 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:28.262 INFO:teuthology.orchestra.run.smithi005.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:28.305 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:28.306 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-26T04:37:28.306 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:28.376 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install rbd-fuse 2024-04-26T04:37:28.595 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:28.673 INFO:teuthology.orchestra.run.smithi111.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:37:28.713 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:28.714 INFO:teuthology.orchestra.run.smithi111.stdout:Nothing to do. 2024-04-26T04:37:28.714 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:28.786 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install rbd-fuse 2024-04-26T04:37:28.910 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:29.035 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:29.062 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:29.062 INFO:teuthology.orchestra.run.smithi005.stdout: Package Architecture Version Repository Size 2024-04-26T04:37:29.062 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 82 k 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 222 k 2024-04-26T04:37:29.063 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:29.303 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:29.368 INFO:teuthology.orchestra.run.smithi005.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 246 kB/s | 82 kB 00:00 2024-04-26T04:37:29.368 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:29.369 INFO:teuthology.orchestra.run.smithi005.stdout:Total 245 kB/s | 82 kB 00:00 2024-04-26T04:37:29.369 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:29.376 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:29.376 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:29.423 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout: Package Architecture Version Repository Size 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:29.424 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 82 k 2024-04-26T04:37:29.425 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 222 k 2024-04-26T04:37:29.425 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:29.444 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:29.444 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:29.595 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:29.695 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:29.756 INFO:teuthology.orchestra.run.smithi111.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 247 kB/s | 82 kB 00:00 2024-04-26T04:37:29.757 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:29.757 INFO:teuthology.orchestra.run.smithi111.stdout:Total 246 kB/s | 82 kB 00:00 2024-04-26T04:37:29.757 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:29.763 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:29.763 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:29.827 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:29.827 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:29.967 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:30.018 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:30.076 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:30.286 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:30.286 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:30.287 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:30.287 INFO:teuthology.orchestra.run.smithi005.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:30.287 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:30.287 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:30.420 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:30.508 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install rbd-mirror 2024-04-26T04:37:30.654 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:30.654 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:30.654 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:30.654 INFO:teuthology.orchestra.run.smithi111.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:30.654 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:30.654 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:30.899 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install rbd-mirror 2024-04-26T04:37:31.044 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:31.169 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:31.170 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:31.170 INFO:teuthology.orchestra.run.smithi005.stdout: Package Arch Version Repo Size 2024-04-26T04:37:31.170 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:31.170 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 2.9 M 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 12 M 2024-04-26T04:37:31.171 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:31.415 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:31.536 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout: Package Arch Version Repo Size 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:37:31.537 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:31.538 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 2.9 M 2024-04-26T04:37:31.538 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 12 M 2024-04-26T04:37:31.538 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:32.253 INFO:teuthology.orchestra.run.smithi111.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 4.1 MB/s | 2.9 MB 00:00 2024-04-26T04:37:32.253 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:32.254 INFO:teuthology.orchestra.run.smithi111.stdout:Total 4.1 MB/s | 2.9 MB 00:00 2024-04-26T04:37:32.254 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:32.263 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:32.263 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:32.342 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:32.342 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:32.408 INFO:teuthology.orchestra.run.smithi005.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 2.4 MB/s | 2.9 MB 00:01 2024-04-26T04:37:32.409 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:32.410 INFO:teuthology.orchestra.run.smithi005.stdout:Total 2.4 MB/s | 2.9 MB 00:01 2024-04-26T04:37:32.410 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:32.422 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:32.422 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:32.508 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:32.509 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:32.754 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:32.812 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:32.835 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:32.835 INFO:teuthology.orchestra.run.smithi111.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:37:32.835 INFO:teuthology.orchestra.run.smithi111.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:37:32.836 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:37:32.836 INFO:teuthology.orchestra.run.smithi111.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:37:32.836 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:32.938 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:33.001 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:33.026 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:33.026 INFO:teuthology.orchestra.run.smithi005.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:37:33.027 INFO:teuthology.orchestra.run.smithi005.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:37:33.027 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:37:33.027 INFO:teuthology.orchestra.run.smithi005.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:37:33.027 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:33.869 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:33.870 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:33.870 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:33.870 INFO:teuthology.orchestra.run.smithi111.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:33.870 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:33.870 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:34.037 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:34.037 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:34.037 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:34.037 INFO:teuthology.orchestra.run.smithi005.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:34.037 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:34.037 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:34.106 DEBUG:teuthology.orchestra.run.smithi111:> sudo yum -y install rbd-nbd 2024-04-26T04:37:34.272 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum -y install rbd-nbd 2024-04-26T04:37:34.621 INFO:teuthology.orchestra.run.smithi111.stdout:Last metadata expiration check: 0:01:43 ago on Fri 26 Apr 2024 04:35:51 AM UTC. 2024-04-26T04:37:34.746 INFO:teuthology.orchestra.run.smithi111.stdout:Dependencies resolved. 2024-04-26T04:37:34.746 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:34.746 INFO:teuthology.orchestra.run.smithi111.stdout: Package Architecture Version Repository Size 2024-04-26T04:37:34.746 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout:Installing: 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction Summary 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout:================================================================================ 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout:Install 1 Package 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:34.747 INFO:teuthology.orchestra.run.smithi111.stdout:Total download size: 165 k 2024-04-26T04:37:34.748 INFO:teuthology.orchestra.run.smithi111.stdout:Installed size: 477 k 2024-04-26T04:37:34.748 INFO:teuthology.orchestra.run.smithi111.stdout:Downloading Packages: 2024-04-26T04:37:34.814 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:44 ago on Fri 26 Apr 2024 04:35:50 AM UTC. 2024-04-26T04:37:34.935 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout: Package Architecture Version Repository Size 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout:Installing: 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction Summary 2024-04-26T04:37:34.936 INFO:teuthology.orchestra.run.smithi005.stdout:================================================================================ 2024-04-26T04:37:34.937 INFO:teuthology.orchestra.run.smithi005.stdout:Install 1 Package 2024-04-26T04:37:34.937 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:34.937 INFO:teuthology.orchestra.run.smithi005.stdout:Total download size: 165 k 2024-04-26T04:37:34.937 INFO:teuthology.orchestra.run.smithi005.stdout:Installed size: 477 k 2024-04-26T04:37:34.937 INFO:teuthology.orchestra.run.smithi005.stdout:Downloading Packages: 2024-04-26T04:37:35.140 INFO:teuthology.orchestra.run.smithi111.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 420 kB/s | 165 kB 00:00 2024-04-26T04:37:35.141 INFO:teuthology.orchestra.run.smithi111.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:35.141 INFO:teuthology.orchestra.run.smithi111.stdout:Total 419 kB/s | 165 kB 00:00 2024-04-26T04:37:35.141 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction check 2024-04-26T04:37:35.149 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction check succeeded. 2024-04-26T04:37:35.149 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction test 2024-04-26T04:37:35.170 INFO:teuthology.orchestra.run.smithi005.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 708 kB/s | 165 kB 00:00 2024-04-26T04:37:35.171 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:37:35.171 INFO:teuthology.orchestra.run.smithi005.stdout:Total 704 kB/s | 165 kB 00:00 2024-04-26T04:37:35.171 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction check 2024-04-26T04:37:35.180 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction check succeeded. 2024-04-26T04:37:35.180 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction test 2024-04-26T04:37:35.211 INFO:teuthology.orchestra.run.smithi111.stdout:Transaction test succeeded. 2024-04-26T04:37:35.211 INFO:teuthology.orchestra.run.smithi111.stdout:Running transaction 2024-04-26T04:37:35.246 INFO:teuthology.orchestra.run.smithi005.stdout:Transaction test succeeded. 2024-04-26T04:37:35.246 INFO:teuthology.orchestra.run.smithi005.stdout:Running transaction 2024-04-26T04:37:35.357 INFO:teuthology.orchestra.run.smithi111.stdout: Preparing : 1/1 2024-04-26T04:37:35.403 INFO:teuthology.orchestra.run.smithi005.stdout: Preparing : 1/1 2024-04-26T04:37:35.460 INFO:teuthology.orchestra.run.smithi111.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:35.493 INFO:teuthology.orchestra.run.smithi005.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:35.880 INFO:teuthology.orchestra.run.smithi111.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:35.886 INFO:teuthology.orchestra.run.smithi005.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:36.112 INFO:teuthology.orchestra.run.smithi111.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:36.112 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:36.112 INFO:teuthology.orchestra.run.smithi111.stdout:Installed: 2024-04-26T04:37:36.112 INFO:teuthology.orchestra.run.smithi111.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:36.112 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:37:36.112 INFO:teuthology.orchestra.run.smithi111.stdout:Complete! 2024-04-26T04:37:36.237 INFO:teuthology.orchestra.run.smithi005.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:37:36.237 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:36.237 INFO:teuthology.orchestra.run.smithi005.stdout:Installed: 2024-04-26T04:37:36.237 INFO:teuthology.orchestra.run.smithi005.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:37:36.237 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:37:36.237 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-26T04:37:36.361 DEBUG:teuthology.parallel:result is None 2024-04-26T04:37:36.474 DEBUG:teuthology.parallel:result is None 2024-04-26T04:37:36.475 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:37:36.475 INFO:teuthology.packaging:ref: None 2024-04-26T04:37:36.475 INFO:teuthology.packaging:tag: None 2024-04-26T04:37:36.475 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:37:36.475 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:37:36.475 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:37:36.705 DEBUG:teuthology.orchestra.run.smithi005:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:37:36.727 INFO:teuthology.orchestra.run.smithi005.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:37:36.727 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:37:36.727 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:37:36.729 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:37:36.729 INFO:teuthology.packaging:ref: None 2024-04-26T04:37:36.729 INFO:teuthology.packaging:tag: None 2024-04-26T04:37:36.729 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:37:36.729 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:37:36.730 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:37:37.272 DEBUG:teuthology.orchestra.run.smithi111:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:37:37.295 INFO:teuthology.orchestra.run.smithi111.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:37:37.295 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:37:37.295 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:37:37.297 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-26T04:37:37.297 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:37:37.298 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:37:37.325 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:37:37.325 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:37:37.363 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-26T04:37:37.363 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:37:37.363 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:37:37.391 DEBUG:teuthology.orchestra.run.smithi005:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:37:37.456 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:37:37.456 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:37:37.482 DEBUG:teuthology.orchestra.run.smithi111:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:37:37.547 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-26T04:37:37.548 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:37:37.548 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:37:37.573 DEBUG:teuthology.orchestra.run.smithi005:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:37:37.637 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:37:37.637 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:37:37.663 DEBUG:teuthology.orchestra.run.smithi111:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:37:37.728 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-26T04:37:37.728 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:37:37.728 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:37:37.752 DEBUG:teuthology.orchestra.run.smithi005:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:37:37.818 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:37:37.818 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:37:37.842 DEBUG:teuthology.orchestra.run.smithi111:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:37:37.907 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T04:37:38.007 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:37:38.007 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:37:38.007 INFO:tasks.cephadm:Cluster fsid is b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:37:38.007 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T04:37:38.007 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.5', 'mon.c': '[v2:172.21.15.5:3301,v1:172.21.15.5:6790]', 'mon.b': '172.21.15.111'} 2024-04-26T04:37:38.008 INFO:tasks.cephadm:First mon is mon.a on smithi005 2024-04-26T04:37:38.008 INFO:tasks.cephadm:First mgr is y 2024-04-26T04:37:38.008 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T04:37:38.008 DEBUG:teuthology.orchestra.run.smithi005:> sudo hostname $(hostname -s) 2024-04-26T04:37:38.033 DEBUG:teuthology.orchestra.run.smithi111:> sudo hostname $(hostname -s) 2024-04-26T04:37:38.057 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-26T04:37:38.057 DEBUG:teuthology.orchestra.run.smithi005:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:37:38.388 INFO:teuthology.orchestra.run.smithi005.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:37 /home/ubuntu/cephtest/cephadm 2024-04-26T04:37:38.388 DEBUG:teuthology.orchestra.run.smithi111:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:37:38.688 INFO:teuthology.orchestra.run.smithi111.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:37 /home/ubuntu/cephtest/cephadm 2024-04-26T04:37:38.688 DEBUG:teuthology.orchestra.run.smithi005:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:37:38.706 DEBUG:teuthology.orchestra.run.smithi111:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:37:38.733 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-26T04:37:38.733 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:37:38.748 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:37:40.225 INFO:teuthology.orchestra.run.smithi005.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:37:40.386 INFO:teuthology.orchestra.run.smithi111.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:38:11.420 INFO:teuthology.orchestra.run.smithi111.stdout:{ 2024-04-26T04:38:11.421 INFO:teuthology.orchestra.run.smithi111.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:38:11.421 INFO:teuthology.orchestra.run.smithi111.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:38:11.421 INFO:teuthology.orchestra.run.smithi111.stdout: "repo_digests": [ 2024-04-26T04:38:11.421 INFO:teuthology.orchestra.run.smithi111.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:38:11.421 INFO:teuthology.orchestra.run.smithi111.stdout: ] 2024-04-26T04:38:11.421 INFO:teuthology.orchestra.run.smithi111.stdout:} 2024-04-26T04:38:13.096 INFO:teuthology.orchestra.run.smithi005.stdout:{ 2024-04-26T04:38:13.096 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:38:13.096 INFO:teuthology.orchestra.run.smithi005.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:38:13.097 INFO:teuthology.orchestra.run.smithi005.stdout: "repo_digests": [ 2024-04-26T04:38:13.097 INFO:teuthology.orchestra.run.smithi005.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:38:13.097 INFO:teuthology.orchestra.run.smithi005.stdout: ] 2024-04-26T04:38:13.097 INFO:teuthology.orchestra.run.smithi005.stdout:} 2024-04-26T04:38:13.118 DEBUG:teuthology.orchestra.run.smithi005:> sudo mkdir -p /etc/ceph 2024-04-26T04:38:13.162 DEBUG:teuthology.orchestra.run.smithi111:> sudo mkdir -p /etc/ceph 2024-04-26T04:38:13.193 DEBUG:teuthology.orchestra.run.smithi005:> sudo chmod 777 /etc/ceph 2024-04-26T04:38:13.227 DEBUG:teuthology.orchestra.run.smithi111:> sudo chmod 777 /etc/ceph 2024-04-26T04:38:13.262 INFO:tasks.cephadm:Writing seed config... 2024-04-26T04:38:13.263 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-26T04:38:13.263 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-26T04:38:13.263 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T04:38:13.263 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T04:38:13.263 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-26T04:38:13.263 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-26T04:38:13.264 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T04:38:13.264 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T04:38:13.264 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T04:38:13.264 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T04:38:13.264 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T04:38:13.264 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:38:13.265 DEBUG:teuthology.orchestra.run.smithi005:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T04:38:13.283 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b5a9bf3e-0386-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T04:38:13.284 DEBUG:teuthology.orchestra.run.smithi005:mon.a> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service 2024-04-26T04:38:13.326 DEBUG:teuthology.orchestra.run.smithi005:mgr.y> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service 2024-04-26T04:38:13.369 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T04:38:13.369 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.5 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:13.580 INFO:teuthology.orchestra.run.smithi005.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:38:13.580 INFO:teuthology.orchestra.run.smithi005.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'b5a9bf3e-0386-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.5', '--skip-admin-label'] 2024-04-26T04:38:13.606 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:38:13.607 INFO:teuthology.orchestra.run.smithi005.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-26T04:38:13.607 INFO:teuthology.orchestra.run.smithi005.stdout:Verifying podman|docker is present... 2024-04-26T04:38:13.631 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:38:13.631 INFO:teuthology.orchestra.run.smithi005.stdout:Verifying lvm2 is present... 2024-04-26T04:38:13.631 INFO:teuthology.orchestra.run.smithi005.stdout:Verifying time synchronization is in place... 2024-04-26T04:38:13.639 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:38:13.639 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:38:13.645 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:38:13.645 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout inactive 2024-04-26T04:38:13.652 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout enabled 2024-04-26T04:38:13.658 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout active 2024-04-26T04:38:13.659 INFO:teuthology.orchestra.run.smithi005.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:38:13.659 INFO:teuthology.orchestra.run.smithi005.stdout:Repeating the final host check... 2024-04-26T04:38:13.682 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:38:13.682 INFO:teuthology.orchestra.run.smithi005.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-26T04:38:13.682 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl is present 2024-04-26T04:38:13.682 INFO:teuthology.orchestra.run.smithi005.stdout:lvcreate is present 2024-04-26T04:38:13.689 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:38:13.689 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:38:13.696 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:38:13.696 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout inactive 2024-04-26T04:38:13.702 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout enabled 2024-04-26T04:38:13.709 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout active 2024-04-26T04:38:13.709 INFO:teuthology.orchestra.run.smithi005.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:38:13.709 INFO:teuthology.orchestra.run.smithi005.stdout:Host looks OK 2024-04-26T04:38:13.709 INFO:teuthology.orchestra.run.smithi005.stdout:Cluster fsid: b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:13.709 INFO:teuthology.orchestra.run.smithi005.stdout:Acquiring lock 140096900284624 on /run/cephadm/b5a9bf3e-0386-11ef-bc93-c7b262605968.lock 2024-04-26T04:38:13.709 INFO:teuthology.orchestra.run.smithi005.stdout:Lock 140096900284624 acquired on /run/cephadm/b5a9bf3e-0386-11ef-bc93-c7b262605968.lock 2024-04-26T04:38:13.710 INFO:teuthology.orchestra.run.smithi005.stdout:Verifying IP 172.21.15.5 port 3300 ... 2024-04-26T04:38:13.710 INFO:teuthology.orchestra.run.smithi005.stdout:Verifying IP 172.21.15.5 port 6789 ... 2024-04-26T04:38:13.710 INFO:teuthology.orchestra.run.smithi005.stdout:Base mon IP(s) is [172.21.15.5:3300, 172.21.15.5:6789], mon addrv is [v2:172.21.15.5:3300,v1:172.21.15.5:6789] 2024-04-26T04:38:13.713 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.5 metric 100 2024-04-26T04:38:13.714 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.5 metric 100 2024-04-26T04:38:13.716 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T04:38:13.716 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-26T04:38:13.716 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-26T04:38:13.719 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T04:38:13.719 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-26T04:38:13.719 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:38:13.719 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-26T04:38:13.719 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:1442/64 scope link noprefixroute 2024-04-26T04:38:13.719 INFO:teuthology.orchestra.run.smithi005.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:38:13.720 INFO:teuthology.orchestra.run.smithi005.stdout:Mon IP `172.21.15.5` is in CIDR network `172.21.0.0/20` 2024-04-26T04:38:13.720 INFO:teuthology.orchestra.run.smithi005.stdout:Mon IP `172.21.15.5` is in CIDR network `172.21.0.0/20` 2024-04-26T04:38:13.720 INFO:teuthology.orchestra.run.smithi005.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-26T04:38:13.720 INFO:teuthology.orchestra.run.smithi005.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T04:38:13.721 INFO:teuthology.orchestra.run.smithi005.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:38:14.201 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stdout 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:38:14.202 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:38:14.202 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stderr Getting image source signatures 2024-04-26T04:38:14.202 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stderr Copying blob sha256:0e2af62137028b073c6ccdb8d4d7e0ccccc475ce5f6e225cb983e90813813045 2024-04-26T04:38:14.202 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stderr Copying blob sha256:3d665f4c032644be9cae748af04bee69f2bfe96cdc3616dabe0ab17899633ab5 2024-04-26T04:38:14.202 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stderr Copying config sha256:0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:38:14.202 INFO:teuthology.orchestra.run.smithi005.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-26T04:38:15.126 INFO:teuthology.orchestra.run.smithi005.stdout:ceph: stdout ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:38:15.126 INFO:teuthology.orchestra.run.smithi005.stdout:Ceph version: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:38:15.126 INFO:teuthology.orchestra.run.smithi005.stdout:Extracting ceph user uid/gid from container image... 2024-04-26T04:38:15.989 INFO:teuthology.orchestra.run.smithi005.stdout:stat: stdout 167 167 2024-04-26T04:38:15.989 INFO:teuthology.orchestra.run.smithi005.stdout:Creating initial keys... 2024-04-26T04:38:16.912 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph-authtool: stdout AQC4LytmkUomEhAAz9LwDLVcFS8eQ2OoIjC1zA== 2024-04-26T04:38:17.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph-authtool: stdout AQC5LytmxoifEBAAyeHf/NWXhhzoSIuvKOCwRA== 2024-04-26T04:38:18.746 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph-authtool: stdout AQC6Lytm+AwiChAAurySgWWZbbL6pLWsgUSJTg== 2024-04-26T04:38:18.747 INFO:teuthology.orchestra.run.smithi005.stdout:Creating initial monmap... 2024-04-26T04:38:19.753 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:monmaptool for a [v2:172.21.15.5:3300,v1:172.21.15.5:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:setting min_mon_release = octopus 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/monmaptool: set fsid to b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:38:19.754 INFO:teuthology.orchestra.run.smithi005.stdout:Creating mon... 2024-04-26T04:38:20.626 INFO:teuthology.orchestra.run.smithi005.stdout:create mon.a on 2024-04-26T04:38:20.850 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-26T04:38:21.003 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T04:38:21.159 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968.target. 2024-04-26T04:38:21.159 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968.target. 2024-04-26T04:38:21.331 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a 2024-04-26T04:38:21.332 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-26T04:38:21.500 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968.target.wants/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@.service. 2024-04-26T04:38:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:21 smithi005 ceph-mon[26000]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:38:22.026 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:38:22.026 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout disabled 2024-04-26T04:38:22.033 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:38:22.033 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout inactive 2024-04-26T04:38:22.033 INFO:teuthology.orchestra.run.smithi005.stdout:firewalld.service is not enabled 2024-04-26T04:38:22.033 INFO:teuthology.orchestra.run.smithi005.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:38:22.033 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for mon to start... 2024-04-26T04:38:22.033 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for mon... 2024-04-26T04:38:22.264 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 bash[25981]: e7a6cb0b8b1a5195e83582b893b21ed9eb40ae1f28d7c1498bd5e6f336f3e10c 2024-04-26T04:38:22.265 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 systemd[1]: Started Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:38:23.115 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout cluster: 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout id: b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout services: 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.608604s) 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout data: 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-26T04:38:23.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-26T04:38:23.117 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-26T04:38:23.117 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout pgs: 2024-04-26T04:38:23.117 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:23.117 INFO:teuthology.orchestra.run.smithi005.stdout:mon is available 2024-04-26T04:38:23.117 INFO:teuthology.orchestra.run.smithi005.stdout:Assimilating anything we can from ceph.conf... 2024-04-26T04:38:23.220 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 ceph-mon[26000]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:38:23.220 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 ceph-mon[26000]: monmap e1: 1 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:38:23.220 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 ceph-mon[26000]: fsmap 2024-04-26T04:38:23.220 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 ceph-mon[26000]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:38:23.220 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 ceph-mon[26000]: mgrmap e1: no daemons active 2024-04-26T04:38:23.221 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:22 smithi005 ceph-mon[26000]: from='client.? 172.21.15.5:0/1521457298' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T04:38:24.115 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout fsid = b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.5:3300,v1:172.21.15.5:6789] 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:38:24.116 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:38:24.117 INFO:teuthology.orchestra.run.smithi005.stdout:Generating new minimal ceph.conf... 2024-04-26T04:38:24.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:23 smithi005 ceph-mon[26000]: from='client.? 172.21.15.5:0/1294678688' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:38:24.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:23 smithi005 ceph-mon[26000]: from='client.? 172.21.15.5:0/1294678688' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T04:38:25.172 INFO:teuthology.orchestra.run.smithi005.stdout:Restarting the monitor... 2024-04-26T04:38:25.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:24 smithi005 ceph-mon[26000]: from='client.? 172.21.15.5:0/1882626974' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:25.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:25 smithi005 systemd[1]: Stopping Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:38:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[25996]: 2024-04-26T04:38:25.437+0000 7f8c592d8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:38:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[25996]: 2024-04-26T04:38:25.437+0000 7f8c592d8700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T04:38:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:25 smithi005 podman[26244]: 2024-04-26 04:38:25.633686467 +0000 UTC m=+0.293069094 container died e7a6cb0b8b1a5195e83582b893b21ed9eb40ae1f28d7c1498bd5e6f336f3e10c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T04:38:26.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:25 smithi005 podman[26244]: 2024-04-26 04:38:25.988257907 +0000 UTC m=+0.647640540 container cleanup e7a6cb0b8b1a5195e83582b893b21ed9eb40ae1f28d7c1498bd5e6f336f3e10c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-26T04:38:26.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:25 smithi005 bash[26244]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:38:26.626 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 podman[26258]: 2024-04-26 04:38:26.267062082 +0000 UTC m=+0.629945647 container remove e7a6cb0b8b1a5195e83582b893b21ed9eb40ae1f28d7c1498bd5e6f336f3e10c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=quincy-c61472c) 2024-04-26T04:38:26.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:38:26.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 systemd[1]: Stopped Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:38:26.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 systemd[1]: Starting Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:38:26.904 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 podman[26353]: 2024-04-26 04:38:26.626113934 +0000 UTC m=+0.104255151 container create 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, RELEASE=quincy-c61472c, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 podman[26353]: 2024-04-26 04:38:26.540177803 +0000 UTC m=+0.018319042 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 podman[26353]: 2024-04-26 04:38:26.820389249 +0000 UTC m=+0.298530468 container init 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c) 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 podman[26353]: 2024-04-26 04:38:26.823384977 +0000 UTC m=+0.301526194 container start 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c) 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: pidfile_write: ignore empty --pid-file 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: load: jerasure load: lrc 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:38:26.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: DB SUMMARY 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: DB Session ID: BYC6DNVTIF7D0IH6O1SM 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84205 ; 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.env: 0x561ad283ab00 2024-04-26T04:38:26.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.fs: Posix File System 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.info_log: 0x561ad44b2ee0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.statistics: (nil) 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.use_fsync: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:38:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:38:26.908 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:38:26.908 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.db_log_dir: 2024-04-26T04:38:26.908 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-26T04:38:26.908 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:38:26.908 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.write_buffer_manager: 0x561ad5222990 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:38:26.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:38:26.910 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:38:26.910 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.unordered_write: 0 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.row_cache: None 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.wal_filter: None 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:38:26.911 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:38:26.912 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:38:26.913 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:38:26.913 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:38:26.913 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:38:26.913 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:38:26.913 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:38:26.913 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_open_files: -1 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:38:26.914 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Compression algorithms supported: 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kZSTD supported: 0 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.merge_operator: 2024-04-26T04:38:26.915 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_filter: None 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561ad4502f58) 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: index_type: 0 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: data_block_index_type: 0 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: index_shortening: 1 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:38:26.916 INFO:journalctl@ceph.mon.a.smithi005.stdout: hash_index_allow_collision: 1 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: checksum: 1 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: no_block_cache: 0 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache: 0x561ad4535090 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache_options: 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: capacity : 536870912 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: num_shard_bits : 4 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: strict_capacity_limit : 0 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache_compressed: (nil) 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: persistent_cache: (nil) 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_size: 4096 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_size_deviation: 10 2024-04-26T04:38:26.917 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_restart_interval: 16 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: index_block_restart_interval: 1 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: metadata_block_size: 4096 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: partition_filters: 0 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: use_delta_encoding: 1 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: whole_key_filtering: 1 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: verify_compression: 0 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: format_version: 4 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: enable_index_compression: 1 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_align: 0 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:38:26.918 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression: NoCompression 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.num_levels: 7 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:38:26.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:38:26.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:38:26.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.ttl: 2592000 2024-04-26T04:38:26.923 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106306849944, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:38:26.924 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106306850724, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80637, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78898, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73107, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106306, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ce1e4591-b70c-4ab2-80a4-35003e9d0be7", "db_session_id": "BYC6DNVTIF7D0IH6O1SM"}} 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106306851062, "job": 1, "event": "recovery_finished"} 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x561ad44c9180 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: DB pointer 0x561ad459e000 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** DB Stats ** 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:38:26.925 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: L0 2/0 80.51 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: Sum 2/0 80.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:38:26.926 INFO:journalctl@ceph.mon.a.smithi005.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative compaction: 0.00 GB write, 24.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval compaction: 0.00 GB write, 24.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:38:26.927 INFO:journalctl@ceph.mon.a.smithi005.stdout: L0 2/0 80.51 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Sum 2/0 80.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 154.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative compaction: 0.00 GB write, 24.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:38:26.928 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: starting mon.a rank 0 at public addrs [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] at bind addrs [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???) e1 preinit fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).mds e1 new map 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).mds e1 print_map 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: e1 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: legacy client fscid: -1 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout: No filesystems configured 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:38:26.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:38:26.930 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:38:26.930 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:38:26.930 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T04:38:26.930 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 bash[26353]: 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 2024-04-26T04:38:26.930 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 systemd[1]: Started Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:38:26.930 INFO:teuthology.orchestra.run.smithi005.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-26T04:38:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:38:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: monmap e1: 1 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:38:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: fsmap 2024-04-26T04:38:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:38:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:26 smithi005 ceph-mon[26371]: mgrmap e1: no daemons active 2024-04-26T04:38:27.947 INFO:teuthology.orchestra.run.smithi005.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-26T04:38:27.948 INFO:teuthology.orchestra.run.smithi005.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:27.948 INFO:teuthology.orchestra.run.smithi005.stdout:Creating mgr... 2024-04-26T04:38:27.948 INFO:teuthology.orchestra.run.smithi005.stdout:Verifying port 9283 ... 2024-04-26T04:38:28.131 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y 2024-04-26T04:38:28.131 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-26T04:38:28.288 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968.target.wants/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@.service. 2024-04-26T04:38:28.656 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:28 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1470559636' entity='client.admin' 2024-04-26T04:38:28.878 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:38:28.878 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout disabled 2024-04-26T04:38:28.886 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:38:28.886 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout inactive 2024-04-26T04:38:28.886 INFO:teuthology.orchestra.run.smithi005.stdout:firewalld.service is not enabled 2024-04-26T04:38:28.886 INFO:teuthology.orchestra.run.smithi005.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:38:28.895 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:38:28.895 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout disabled 2024-04-26T04:38:28.903 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:38:28.903 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout inactive 2024-04-26T04:38:28.903 INFO:teuthology.orchestra.run.smithi005.stdout:firewalld.service is not enabled 2024-04-26T04:38:28.903 INFO:teuthology.orchestra.run.smithi005.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T04:38:28.903 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for mgr to start... 2024-04-26T04:38:28.904 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for mgr... 2024-04-26T04:38:29.030 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:28 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:28.913+0000 7f542d6611c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:38:29.326 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:29 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:29.084+0000 7f542d6611c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:29.326 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:29 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:29.191+0000 7f542d6611c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:38:29.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:29 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/911413422' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:38:29.605 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:29 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:29.519+0000 7f542d6611c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:38:29.605 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:29 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:29.603+0000 7f542d6611c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsid": "b5a9bf3e-0386-11ef-bc93-c7b262605968", 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:38:29.972 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.973 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:38:29.974 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:38:29.975 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:38:21.943092+0000", 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:38:29.976 INFO:teuthology.orchestra.run.smithi005.stdout:mgr not available, waiting (1/15)... 2024-04-26T04:38:31.160 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:30.819+0000 7f542d6611c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:38:31.160 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:30.987+0000 7f542d6611c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:31.160 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:31.158+0000 7f542d6611c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:38:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:31.425+0000 7f542d6611c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:38:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:31.517+0000 7f542d6611c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:38:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:31.612+0000 7f542d6611c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:38:32.203 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:31.894+0000 7f542d6611c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:32.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:32 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:32.382+0000 7f542d6611c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:38:32.945 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:32 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3462059608' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:38:33.087 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:33.087 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:38:33.087 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsid": "b5a9bf3e-0386-11ef-bc93-c7b262605968", 2024-04-26T04:38:33.087 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:38:33.087 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:38:33.088 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:38:33.089 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:38:33.090 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:38:21.943092+0000", 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:38:33.091 INFO:teuthology.orchestra.run.smithi005.stdout:mgr not available, waiting (2/15)... 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:32 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:32.943+0000 7f542d6611c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:33.039+0000 7f542d6611c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:33.222+0000 7f542d6611c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:38:33.787 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:33.521+0000 7f542d6611c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:38:33.787 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:33.785+0000 7f542d6611c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:38:34.165 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:34.029+0000 7f542d6611c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:38:34.165 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:34.164+0000 7f542d6611c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:38:34.465 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:34.258+0000 7f542d6611c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:38:34.465 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:34.374+0000 7f542d6611c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:34.465 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:34.464+0000 7f542d6611c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:38:35.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:35.292+0000 7f542d6611c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:38:35.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:35.491+0000 7f542d6611c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: Activating manager daemon y 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: mgrmap e2: y(active, starting, since 0.00309221s) 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: Manager daemon y is now available 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' 2024-04-26T04:38:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' 2024-04-26T04:38:35.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:35 smithi005 ceph-mon[26371]: from='mgr.14100 172.21.15.5:0/1123983410' entity='mgr.y' 2024-04-26T04:38:36.164 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsid": "b5a9bf3e-0386-11ef-bc93-c7b262605968", 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:38:36.165 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:38:36.166 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:38:36.167 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:38:36.170 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:36.171 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:38:21.943092+0000", 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:38:36.172 INFO:teuthology.orchestra.run.smithi005.stdout:mgr not available, waiting (3/15)... 2024-04-26T04:38:36.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:36 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2624101227' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:38:36.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:36 smithi005 ceph-mon[26371]: mgrmap e3: y(active, since 1.00715s) 2024-04-26T04:38:38.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:38 smithi005 ceph-mon[26371]: mgrmap e4: y(active, since 2s) 2024-04-26T04:38:41.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:41 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2756048015' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:38:47.848 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:47.848 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:38:47.848 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsid": "b5a9bf3e-0386-11ef-bc93-c7b262605968", 2024-04-26T04:38:47.848 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:47.849 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:38:47.850 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:38:47.854 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:38:47.855 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:38:21.943092+0000", 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:38:47.856 INFO:teuthology.orchestra.run.smithi005.stdout:mgr is available 2024-04-26T04:38:49.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:49 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3103870742' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:38:49.496 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout fsid = b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:38:49.497 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:38:49.498 INFO:teuthology.orchestra.run.smithi005.stdout:Enabling cephadm module... 2024-04-26T04:38:50.561 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1554638130' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T04:38:50.562 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setuser ceph since I am not root 2024-04-26T04:38:50.562 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setgroup ceph since I am not root 2024-04-26T04:38:50.562 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:50.393+0000 7f477dcbd1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:38:50.562 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:50.560+0000 7f477dcbd1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:50.824 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:50.663+0000 7f477dcbd1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:38:51.169 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:50.984+0000 7f477dcbd1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:38:51.169 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:51 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:51.071+0000 7f477dcbd1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:38:51.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:51 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1554638130' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T04:38:51.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:51 smithi005 ceph-mon[26371]: mgrmap e5: y(active, since 14s) 2024-04-26T04:38:52.152 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:38:52.152 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-26T04:38:52.152 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:38:52.152 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:38:52.152 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:38:52.153 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:38:52.153 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for the mgr to restart... 2024-04-26T04:38:52.153 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for mgr epoch 5... 2024-04-26T04:38:52.425 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:52 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/131511167' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:38:52.425 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:52.284+0000 7f477dcbd1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:38:52.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:52.457+0000 7f477dcbd1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:52.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:52.631+0000 7f477dcbd1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:38:53.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:52.899+0000 7f477dcbd1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:38:53.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:52.991+0000 7f477dcbd1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:38:53.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:53.087+0000 7f477dcbd1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:38:53.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:53.369+0000 7f477dcbd1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:54.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:53.862+0000 7f477dcbd1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:38:54.682 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:54.404+0000 7f477dcbd1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:38:54.682 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:54.500+0000 7f477dcbd1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:38:54.682 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:54.680+0000 7f477dcbd1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:38:55.243 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:54.980+0000 7f477dcbd1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:38:55.243 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:55.242+0000 7f477dcbd1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:38:55.621 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:55.485+0000 7f477dcbd1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:38:55.621 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:55.620+0000 7f477dcbd1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:38:55.922 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:55.714+0000 7f477dcbd1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:38:55.923 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:55.831+0000 7f477dcbd1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:55.923 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:55.921+0000 7f477dcbd1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:38:57.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:56.736+0000 7f477dcbd1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:38:57.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:38:56.931+0000 7f477dcbd1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: Active manager daemon y restarted 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: Activating manager daemon y 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: mgrmap e6: y(active, starting, since 0.00315251s) 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:38:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: Manager daemon y is now available 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:57.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:56 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:57 smithi005 ceph-mon[26371]: Found migration_current of "None". Setting to last migration. 2024-04-26T04:38:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:57 smithi005 ceph-mon[26371]: mgrmap e7: y(active, since 1.00525s) 2024-04-26T04:38:58.558 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:38:58.558 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-26T04:38:58.558 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:38:58.558 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:38:58.558 INFO:teuthology.orchestra.run.smithi005.stdout:mgr epoch 5 is available 2024-04-26T04:38:58.558 INFO:teuthology.orchestra.run.smithi005.stdout:Setting orchestrator backend to cephadm... 2024-04-26T04:38:59.154 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:58 smithi005 ceph-mon[26371]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:38:59.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:58 smithi005 ceph-mon[26371]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:39:00.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:59 smithi005 ceph-mon[26371]: mgrmap e8: y(active, since 2s) 2024-04-26T04:39:00.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:59 smithi005 ceph-mon[26371]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:00.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:59 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:00.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:38:59 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:00.837 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-26T04:39:00.837 INFO:teuthology.orchestra.run.smithi005.stdout:Generating ssh key... 2024-04-26T04:39:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-mon[26371]: [26/Apr/2024:04:38:59] ENGINE Bus STARTING 2024-04-26T04:39:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-mon[26371]: [26/Apr/2024:04:39:00] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:39:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-mon[26371]: [26/Apr/2024:04:39:00] ENGINE Bus STARTED 2024-04-26T04:39:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: Generating public/private rsa key pair. 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: Your identification has been saved in /tmp/tmp_xohwzrn/key. 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: Your public key has been saved in /tmp/tmp_xohwzrn/key.pub. 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: The key fingerprint is: 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: SHA256:7TU3qY1b2ByntvALrQnqr1Nyz3dC+6L7T90a+xN19CQ ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: The key's randomart image is: 2024-04-26T04:39:01.882 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: +---[RSA 3072]----+ 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | | 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | E o| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | +.| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | . .+| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | S . o = +| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | ..o. @.*o| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | +oo*.X.=| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | .. .oX=B.| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: | .++. *=BO=| 2024-04-26T04:39:01.883 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: +----[SHA256]-----+ 2024-04-26T04:39:02.157 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:02 smithi005 ceph-mon[26371]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:02.157 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:02 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:02.158 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:02 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:03.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:03 smithi005 ceph-mon[26371]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:03.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:03 smithi005 ceph-mon[26371]: Generating ssh key... 2024-04-26T04:39:03.392 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgnzRH+ilZzDYI//fe7lQfBcHVS++ZOz5u8erm8GvrVHulffohqOhn9nxGL4Osec1P12gOxAAGqebMwFGO8CYMgBp4sH/smYwsxKO43clUSgOEd5AA2EWDJJZCj1kFM+MxNUESoPxe491ILRgcXvEZzRDnoGr6XWGJWT4axw7LgnxJWHziam/3FulRM3TQ7e6JDfd5g30t6odvj7pePQZ3qpcAxDKE/UyeZHNaR4Nucv5EsLgBn5qu1pquD0zlRDm2STGvqU78ebhvPktEDb8NVkGRXR2Fhco61SzDbyc8+SYF9cFUTqaLEmGRmH12+1PJa53gCYEgx6S27E1i4FAM04M3G42TmRcXDrdajprwjDq0ErIsHdT+s7yGAZKglW+OAhVrOe3ExAt2FOzWixEpAxTz2AlpI6xC3z3qyKYOH4pBboZIpmOIC9xWi5ezNOZTybTybQaAO5rAV9HAga7aLEQ3t0ml75h43RsqOutGQiqA55IVxx3JbOtt91L4yYU= ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:39:03.393 INFO:teuthology.orchestra.run.smithi005.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:39:03.393 INFO:teuthology.orchestra.run.smithi005.stdout:Adding key to root@localhost authorized_keys... 2024-04-26T04:39:03.393 INFO:teuthology.orchestra.run.smithi005.stdout:Adding host smithi005... 2024-04-26T04:39:04.309 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:04 smithi005 ceph-mon[26371]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:05.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:05 smithi005 ceph-mon[26371]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi005", "addr": "172.21.15.5", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:06.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:06 smithi005 ceph-mon[26371]: Deploying cephadm binary to smithi005 2024-04-26T04:39:06.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:06 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:06.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:06 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.442 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout Added host 'smithi005' with addr '172.21.15.5' 2024-04-26T04:39:06.442 INFO:teuthology.orchestra.run.smithi005.stdout:Deploying unmanaged mon service... 2024-04-26T04:39:07.078 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:07 smithi005 ceph-mon[26371]: Added host smithi005 2024-04-26T04:39:07.801 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-26T04:39:07.801 INFO:teuthology.orchestra.run.smithi005.stdout:Deploying unmanaged mgr service... 2024-04-26T04:39:08.368 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:08 smithi005 ceph-mon[26371]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:08.368 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:08 smithi005 ceph-mon[26371]: Saving service mon spec with placement count:5 2024-04-26T04:39:08.368 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:08 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:09.814 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-26T04:39:10.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:09 smithi005 ceph-mon[26371]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:10.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:09 smithi005 ceph-mon[26371]: Saving service mgr spec with placement count:2 2024-04-26T04:39:10.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:09 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:10.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:09 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:10 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:10 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1084727835' entity='client.admin' 2024-04-26T04:39:12.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:12 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:12.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:12 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1290565579' entity='client.admin' 2024-04-26T04:39:12.876 INFO:teuthology.orchestra.run.smithi005.stdout:Enabling the dashboard module... 2024-04-26T04:39:14.171 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-mon[26371]: from='mgr.14120 172.21.15.5:0/96416559' entity='mgr.y' 2024-04-26T04:39:14.171 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2395861047' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T04:39:14.473 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setuser ceph since I am not root 2024-04-26T04:39:14.473 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setgroup ceph since I am not root 2024-04-26T04:39:14.473 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:14.327+0000 7fee84fb91c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:39:14.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:14.496+0000 7fee84fb91c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:39:14.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:14.599+0000 7fee84fb91c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:39:15.094 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:14.923+0000 7fee84fb91c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:39:15.094 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:15 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:15.009+0000 7fee84fb91c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:39:15.426 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:15 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2395861047' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T04:39:15.427 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:15 smithi005 ceph-mon[26371]: mgrmap e9: y(active, since 17s) 2024-04-26T04:39:16.003 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:39:16.003 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-26T04:39:16.004 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:39:16.004 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:39:16.004 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:39:16.004 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:39:16.004 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for the mgr to restart... 2024-04-26T04:39:16.004 INFO:teuthology.orchestra.run.smithi005.stdout:Waiting for mgr epoch 9... 2024-04-26T04:39:16.432 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:16 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/579246392' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:39:16.432 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:16.255+0000 7fee84fb91c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:39:16.432 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:16.430+0000 7fee84fb91c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:39:16.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:16.606+0000 7fee84fb91c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:39:17.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:16.877+0000 7fee84fb91c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:39:17.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:16.970+0000 7fee84fb91c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:39:17.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:17.067+0000 7fee84fb91c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:39:17.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:17.352+0000 7fee84fb91c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:39:18.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:17.833+0000 7fee84fb91c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:39:18.656 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:18 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:18.375+0000 7fee84fb91c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:39:18.656 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:18 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:18.471+0000 7fee84fb91c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:39:18.656 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:18 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:18.655+0000 7fee84fb91c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:39:19.224 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:18 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:18.958+0000 7fee84fb91c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:39:19.224 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:19.222+0000 7fee84fb91c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:39:19.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:19.469+0000 7fee84fb91c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:39:19.914 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:19.605+0000 7fee84fb91c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:39:19.914 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:19.701+0000 7fee84fb91c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:39:19.914 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:19.820+0000 7fee84fb91c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:39:19.915 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:19.912+0000 7fee84fb91c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:39:20.989 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:20.737+0000 7fee84fb91c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:39:20.989 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:39:20.935+0000 7fee84fb91c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: Active manager daemon y restarted 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: Activating manager daemon y 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: mgrmap e10: y(active, starting, since 0.00317457s) 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:39:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: Manager daemon y is now available 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:21.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:22.546 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:39:22.546 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-26T04:39:22.547 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:39:22.547 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:39:22.547 INFO:teuthology.orchestra.run.smithi005.stdout:mgr epoch 9 is available 2024-04-26T04:39:22.547 INFO:teuthology.orchestra.run.smithi005.stdout:Generating a dashboard self-signed certificate... 2024-04-26T04:39:22.961 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: [26/Apr/2024:04:39:21] ENGINE Bus STARTING 2024-04-26T04:39:22.961 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: [26/Apr/2024:04:39:21] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:39:22.961 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: [26/Apr/2024:04:39:21] ENGINE Bus STARTED 2024-04-26T04:39:22.961 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:22.962 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:39:22.962 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: mgrmap e11: y(active, since 1.0055s) 2024-04-26T04:39:22.962 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:22 smithi005 ceph-mon[26371]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:39:23.861 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-26T04:39:23.861 INFO:teuthology.orchestra.run.smithi005.stdout:Creating initial admin user... 2024-04-26T04:39:24.186 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:24 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:24.186 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:24 smithi005 ceph-mon[26371]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:24.186 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:24 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:24.187 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:24 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:24.187 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:24 smithi005 ceph-mon[26371]: mgrmap e12: y(active, since 2s) 2024-04-26T04:39:25.294 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Cd6vU64Go3Y4nzqbIbDYA.9BHH45ZIyS8uSKiUWxLvsTp7Vq0n59W", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714106364, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T04:39:25.294 INFO:teuthology.orchestra.run.smithi005.stdout:Fetching dashboard port number... 2024-04-26T04:39:25.712 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:25 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:25.712 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:25 smithi005 ceph-mon[26371]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:25.712 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:25 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:26.633 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:26 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2494678748' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T04:39:27.256 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stdout 8443 2024-04-26T04:39:27.268 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:39:27.268 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout disabled 2024-04-26T04:39:27.278 INFO:teuthology.orchestra.run.smithi005.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:39:27.278 INFO:teuthology.orchestra.run.smithi005.stdout:systemctl: stdout inactive 2024-04-26T04:39:27.278 INFO:teuthology.orchestra.run.smithi005.stdout:firewalld.service is not enabled 2024-04-26T04:39:27.278 INFO:teuthology.orchestra.run.smithi005.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T04:39:27.279 INFO:teuthology.orchestra.run.smithi005.stdout:Ceph Dashboard is now available at: 2024-04-26T04:39:27.279 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:27.279 INFO:teuthology.orchestra.run.smithi005.stdout: URL: https://smithi005:8443/ 2024-04-26T04:39:27.279 INFO:teuthology.orchestra.run.smithi005.stdout: User: admin 2024-04-26T04:39:27.280 INFO:teuthology.orchestra.run.smithi005.stdout: Password: 2yahzsbszu 2024-04-26T04:39:27.280 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:27.280 INFO:teuthology.orchestra.run.smithi005.stdout:Saving cluster configuration to /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config directory 2024-04-26T04:39:27.280 INFO:teuthology.orchestra.run.smithi005.stdout:Enabling autotune for osd_memory_target 2024-04-26T04:39:28.823 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:28 smithi005 ceph-mon[26371]: mgrmap e13: y(active, since 6s) 2024-04-26T04:39:29.729 INFO:teuthology.orchestra.run.smithi005.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-26T04:39:29.730 INFO:teuthology.orchestra.run.smithi005.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-26T04:39:29.730 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.730 INFO:teuthology.orchestra.run.smithi005.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:29.730 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout:Or, if you are only running a single cluster on this host: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: ceph telemetry on 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout:For more information see: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:29.731 INFO:teuthology.orchestra.run.smithi005.stdout:Bootstrap complete. 2024-04-26T04:39:29.771 INFO:tasks.cephadm:Fetching config... 2024-04-26T04:39:29.772 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:39:29.772 DEBUG:teuthology.orchestra.run.smithi005:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T04:39:29.789 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T04:39:29.789 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:39:29.789 DEBUG:teuthology.orchestra.run.smithi005:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T04:39:29.845 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T04:39:29.846 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:39:29.846 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd if=/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-26T04:39:29.911 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T04:39:29.911 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:39:29.911 DEBUG:teuthology.orchestra.run.smithi005:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T04:39:29.966 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T04:39:29.967 DEBUG:teuthology.orchestra.run.smithi005:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgnzRH+ilZzDYI//fe7lQfBcHVS++ZOz5u8erm8GvrVHulffohqOhn9nxGL4Osec1P12gOxAAGqebMwFGO8CYMgBp4sH/smYwsxKO43clUSgOEd5AA2EWDJJZCj1kFM+MxNUESoPxe491ILRgcXvEZzRDnoGr6XWGJWT4axw7LgnxJWHziam/3FulRM3TQ7e6JDfd5g30t6odvj7pePQZ3qpcAxDKE/UyeZHNaR4Nucv5EsLgBn5qu1pquD0zlRDm2STGvqU78ebhvPktEDb8NVkGRXR2Fhco61SzDbyc8+SYF9cFUTqaLEmGRmH12+1PJa53gCYEgx6S27E1i4FAM04M3G42TmRcXDrdajprwjDq0ErIsHdT+s7yGAZKglW+OAhVrOe3ExAt2FOzWixEpAxTz2AlpI6xC3z3qyKYOH4pBboZIpmOIC9xWi5ezNOZTybTybQaAO5rAV9HAga7aLEQ3t0ml75h43RsqOutGQiqA55IVxx3JbOtt91L4yYU= ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:39:30.053 INFO:teuthology.orchestra.run.smithi005.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgnzRH+ilZzDYI//fe7lQfBcHVS++ZOz5u8erm8GvrVHulffohqOhn9nxGL4Osec1P12gOxAAGqebMwFGO8CYMgBp4sH/smYwsxKO43clUSgOEd5AA2EWDJJZCj1kFM+MxNUESoPxe491ILRgcXvEZzRDnoGr6XWGJWT4axw7LgnxJWHziam/3FulRM3TQ7e6JDfd5g30t6odvj7pePQZ3qpcAxDKE/UyeZHNaR4Nucv5EsLgBn5qu1pquD0zlRDm2STGvqU78ebhvPktEDb8NVkGRXR2Fhco61SzDbyc8+SYF9cFUTqaLEmGRmH12+1PJa53gCYEgx6S27E1i4FAM04M3G42TmRcXDrdajprwjDq0ErIsHdT+s7yGAZKglW+OAhVrOe3ExAt2FOzWixEpAxTz2AlpI6xC3z3qyKYOH4pBboZIpmOIC9xWi5ezNOZTybTybQaAO5rAV9HAga7aLEQ3t0ml75h43RsqOutGQiqA55IVxx3JbOtt91L4yYU= ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:39:30.067 DEBUG:teuthology.orchestra.run.smithi111:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgnzRH+ilZzDYI//fe7lQfBcHVS++ZOz5u8erm8GvrVHulffohqOhn9nxGL4Osec1P12gOxAAGqebMwFGO8CYMgBp4sH/smYwsxKO43clUSgOEd5AA2EWDJJZCj1kFM+MxNUESoPxe491ILRgcXvEZzRDnoGr6XWGJWT4axw7LgnxJWHziam/3FulRM3TQ7e6JDfd5g30t6odvj7pePQZ3qpcAxDKE/UyeZHNaR4Nucv5EsLgBn5qu1pquD0zlRDm2STGvqU78ebhvPktEDb8NVkGRXR2Fhco61SzDbyc8+SYF9cFUTqaLEmGRmH12+1PJa53gCYEgx6S27E1i4FAM04M3G42TmRcXDrdajprwjDq0ErIsHdT+s7yGAZKglW+OAhVrOe3ExAt2FOzWixEpAxTz2AlpI6xC3z3qyKYOH4pBboZIpmOIC9xWi5ezNOZTybTybQaAO5rAV9HAga7aLEQ3t0ml75h43RsqOutGQiqA55IVxx3JbOtt91L4yYU= ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:39:30.106 INFO:teuthology.orchestra.run.smithi111.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgnzRH+ilZzDYI//fe7lQfBcHVS++ZOz5u8erm8GvrVHulffohqOhn9nxGL4Osec1P12gOxAAGqebMwFGO8CYMgBp4sH/smYwsxKO43clUSgOEd5AA2EWDJJZCj1kFM+MxNUESoPxe491ILRgcXvEZzRDnoGr6XWGJWT4axw7LgnxJWHziam/3FulRM3TQ7e6JDfd5g30t6odvj7pePQZ3qpcAxDKE/UyeZHNaR4Nucv5EsLgBn5qu1pquD0zlRDm2STGvqU78ebhvPktEDb8NVkGRXR2Fhco61SzDbyc8+SYF9cFUTqaLEmGRmH12+1PJa53gCYEgx6S27E1i4FAM04M3G42TmRcXDrdajprwjDq0ErIsHdT+s7yGAZKglW+OAhVrOe3ExAt2FOzWixEpAxTz2AlpI6xC3z3qyKYOH4pBboZIpmOIC9xWi5ezNOZTybTybQaAO5rAV9HAga7aLEQ3t0ml75h43RsqOutGQiqA55IVxx3JbOtt91L4yYU= ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:39:30.118 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T04:39:30.382 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:30.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:30 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2759240234' entity='client.admin' 2024-04-26T04:39:33.392 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T04:39:33.392 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T04:39:33.656 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:34.029 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:33 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3056093524' entity='client.admin' 2024-04-26T04:39:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi005", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:36.632 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi111 2024-04-26T04:39:36.632 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:39:36.632 DEBUG:teuthology.orchestra.run.smithi111:> dd of=/etc/ceph/ceph.conf 2024-04-26T04:39:36.650 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:39:36.650 DEBUG:teuthology.orchestra.run.smithi111:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:36.709 INFO:tasks.cephadm:Adding host smithi111 to orchestrator... 2024-04-26T04:39:36.709 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch host add smithi111 2024-04-26T04:39:36.968 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:37.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:37.191 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:38.154 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:37 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:38.154 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:37 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:39.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:39 smithi005 ceph-mon[26371]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi111", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:40.597 INFO:teuthology.orchestra.run.smithi005.stdout:Added host 'smithi111' with addr '172.21.15.111' 2024-04-26T04:39:40.597 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:40 smithi005 ceph-mon[26371]: Deploying cephadm binary to smithi111 2024-04-26T04:39:41.222 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T04:39:41.477 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:41.887 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:41.888 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:41 smithi005 ceph-mon[26371]: Added host smithi111 2024-04-26T04:39:41.888 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:41.888 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:41.888 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:41 smithi005 ceph-mon[26371]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:41.888 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:43.048 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:39:43.048 INFO:teuthology.orchestra.run.smithi005.stdout:[{"addr": "172.21.15.5", "hostname": "smithi005", "labels": [], "status": ""}, {"addr": "172.21.15.111", "hostname": "smithi111", "labels": [], "status": ""}] 2024-04-26T04:39:43.089 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:43.598 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T04:39:43.599 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T04:39:43.857 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:44.167 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:43 smithi005 ceph-mon[26371]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:44.167 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:43 smithi005 ceph-mon[26371]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:45.847 INFO:teuthology.orchestra.run.smithi005.stderr:adjusted tunables profile to default 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi111", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:46.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4001438578' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.153 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:46.456 INFO:tasks.cephadm:Adding mon.a on smithi005 2024-04-26T04:39:46.456 INFO:tasks.cephadm:Adding mon.c on smithi005 2024-04-26T04:39:46.456 INFO:tasks.cephadm:Adding mon.b on smithi111 2024-04-26T04:39:46.456 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi005:172.21.15.5=a;smithi005:[v2:172.21.15.5:3301,v1:172.21.15.5:6790]=c;smithi111:172.21.15.111=b' 2024-04-26T04:39:46.716 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:46 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:46 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:46 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4001438578' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:39:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:46 smithi005 ceph-mon[26371]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:39:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:47 smithi005 ceph-mon[26371]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:48.319 INFO:teuthology.orchestra.run.smithi111.stdout:Scheduled mon update... 2024-04-26T04:39:48.881 DEBUG:teuthology.orchestra.run.smithi005:mon.c> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.c.service 2024-04-26T04:39:48.884 DEBUG:teuthology.orchestra.run.smithi111:mon.b> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.b.service 2024-04-26T04:39:48.886 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:39:48.886 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:39:49.627 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi005:172.21.15.5=a;smithi005:[v2:172.21.15.5:3301,v1:172.21.15.5:6790]=c;smithi111:172.21.15.111=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: Saving service mon spec with placement smithi005:172.21.15.5=a;smithi005:[v2:172.21.15.5:3301,v1:172.21.15.5:6790]=c;smithi111:172.21.15.111=b;count:3 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:39:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:49.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: Deploying daemon mon.b on smithi111 2024-04-26T04:39:49.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:49 smithi005 ceph-mon[26371]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:52.245 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 systemd[1]: Starting Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:39:52.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:51 smithi005 ceph-mon[26371]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 podman[30202]: 2024-04-26 04:39:52.638098246 +0000 UTC m=+0.249156571 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:39:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 podman[30202]: 2024-04-26 04:39:52.739008031 +0000 UTC m=+0.350066331 container create 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-26T04:39:52.884 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:39:52.884 INFO:teuthology.orchestra.run.smithi111.stdout:{"epoch":1,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","modified":"2024-04-26T04:38:19.110602Z","created":"2024-04-26T04:38:19.110602Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:3300","nonce":0},{"type":"v1","addr":"172.21.15.5:6789","nonce":0}]},"addr":"172.21.15.5:6789/0","public_addr":"172.21.15.5:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T04:39:52.886 INFO:teuthology.orchestra.run.smithi111.stderr:dumped monmap epoch 1 2024-04-26T04:39:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 podman[30202]: 2024-04-26 04:39:52.958414204 +0000 UTC m=+0.569472504 container init 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 podman[30202]: 2024-04-26 04:39:52.96214263 +0000 UTC m=+0.573200931 container start 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: pidfile_write: ignore empty --pid-file 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: load: jerasure load: lrc 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: DB SUMMARY 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: DB Session ID: DU5FVPOXXORAEWPCNQQA 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:39:53.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.env: 0x55cd61cb0b00 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.fs: Posix File System 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.info_log: 0x55cd62c7aee0 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.statistics: (nil) 2024-04-26T04:39:53.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.use_fsync: 0 2024-04-26T04:39:53.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:39:53.272 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:39:53.272 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:39:53.272 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:39:53.272 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:39:53.272 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:39:53.272 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.db_log_dir: 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:39:53.273 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.write_buffer_manager: 0x55cd639ea990 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.unordered_write: 0 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:39:53.274 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.row_cache: None 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.wal_filter: None 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:39:53.275 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:39:53.276 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_open_files: -1 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:39:53.277 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Compression algorithms supported: 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kZSTD supported: 0 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:39:53.278 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.merge_operator: 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_filter: None 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cd62ccaf58) 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: index_type: 0 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: data_block_index_type: 0 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: index_shortening: 1 2024-04-26T04:39:53.279 INFO:journalctl@ceph.mon.b.smithi111.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: hash_index_allow_collision: 1 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: checksum: 1 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: no_block_cache: 0 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache: 0x55cd62cfd090 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache_options: 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: capacity : 536870912 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: num_shard_bits : 4 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: strict_capacity_limit : 0 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache_compressed: (nil) 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: persistent_cache: (nil) 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_size: 4096 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_size_deviation: 10 2024-04-26T04:39:53.280 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_restart_interval: 16 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: index_block_restart_interval: 1 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: metadata_block_size: 4096 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: partition_filters: 0 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: use_delta_encoding: 1 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: whole_key_filtering: 1 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: verify_compression: 0 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: format_version: 4 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: enable_index_compression: 1 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_align: 0 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:39:53.281 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression: NoCompression 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.num_levels: 7 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:39:53.282 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:39:53.283 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:39:53.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:39:53.285 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.ttl: 2592000 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:39:53.286 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106392989047, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106392989350, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106392, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "6e051ac3-38ed-4fe1-bf01-780d72ab17ee", "db_session_id": "DU5FVPOXXORAEWPCNQQA"}} 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:39:53.287 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106392989654, "job": 1, "event": "recovery_finished"} 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55cd62c91180 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: DB pointer 0x55cd62d66000 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** DB Stats ** 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:39:53.288 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** Compaction Stats [default] ** 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** Compaction Stats [default] ** 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:39:53.289 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** Compaction Stats [default] ** 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:39:53.290 INFO:journalctl@ceph.mon.b.smithi111.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** Compaction Stats [default] ** 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:39:53.291 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: using public_addr v2:172.21.15.111:0/0 -> [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: starting mon.b rank -1 at public addrs [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] at bind addrs [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:52 smithi111 ceph-mon[30255]: mon.b@-1(???) e0 preinit fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).mds e1 new map 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: e1 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: legacy client fscid: -1 2024-04-26T04:39:53.292 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout: No filesystems configured 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.293 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi111", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4001438578' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:53.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4001438578' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 bash[30202]: 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi005:172.21.15.5=a;smithi005:[v2:172.21.15.5:3301,v1:172.21.15.5:6790]=c;smithi111:172.21.15.111=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: Saving service mon spec with placement smithi005:172.21.15.5=a;smithi005:[v2:172.21.15.5:3301,v1:172.21.15.5:6790]=c;smithi111:172.21.15.111=b;count:3 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: Deploying daemon mon.b on smithi111 2024-04-26T04:39:53.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:53.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:53.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 ceph-mon[30255]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:39:53.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:53 smithi111 systemd[1]: Started Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:39:53.367 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:53 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/3317616591' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:39:54.586 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:39:54.586 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:39:54.841 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: mon.a calling monitor election 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: mon.b calling monitor election 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: monmap e2: 2 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0],b=[v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: fsmap 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: mgrmap e13: y(active, since 37s) 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: overall HEALTH_OK 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:58.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:58.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/3317616591' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:39:58.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: mon.a calling monitor election 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: mon.b calling monitor election 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: monmap e2: 2 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0],b=[v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: fsmap 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: mgrmap e13: y(active, since 37s) 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: overall HEALTH_OK 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:58.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:59.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:59 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:59.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:59 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:39:59.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:59 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:59.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:59 smithi005 ceph-mon[26371]: Deploying daemon mon.c on smithi005 2024-04-26T04:39:59.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:39:59 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:59 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:39:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:59 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:39:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:59 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:59 smithi111 ceph-mon[30255]: Deploying daemon mon.c on smithi005 2024-04-26T04:39:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:39:59 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:59.580 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:39:59.581 INFO:teuthology.orchestra.run.smithi111.stdout:{"epoch":2,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","modified":"2024-04-26T04:39:53.012388Z","created":"2024-04-26T04:38:19.110602Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:3300","nonce":0},{"type":"v1","addr":"172.21.15.5:6789","nonce":0}]},"addr":"172.21.15.5:6789/0","public_addr":"172.21.15.5:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:3300","nonce":0},{"type":"v1","addr":"172.21.15.111:6789","nonce":0}]},"addr":"172.21.15.111:6789/0","public_addr":"172.21.15.111:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T04:39:59.582 INFO:teuthology.orchestra.run.smithi111.stderr:dumped monmap epoch 2 2024-04-26T04:40:00.124 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:00 smithi111 ceph-mon[30255]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:00.124 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:00 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/4257832461' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:40:00.124 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:00 smithi111 ceph-mon[30255]: overall HEALTH_OK 2024-04-26T04:40:00.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:00 smithi005 ceph-mon[26371]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:00.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:00 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/4257832461' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:40:00.440 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:00 smithi005 ceph-mon[26371]: overall HEALTH_OK 2024-04-26T04:40:01.148 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:40:01.149 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:40:01.402 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:40:01.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:01 smithi005 systemd[1]: Started Ceph mon.c for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:06.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:40:06.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: mon.a calling monitor election 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: mon.b calling monitor election 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: mon.c calling monitor election 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: monmap e3: 3 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0],b=[v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0],c=[v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: fsmap 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: mgrmap e13: y(active, since 45s) 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: overall HEALTH_OK 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: mon.a calling monitor election 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: mon.b calling monitor election 2024-04-26T04:40:06.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: mon.c calling monitor election 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: monmap e3: 3 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0],b=[v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0],c=[v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: fsmap 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: mgrmap e13: y(active, since 45s) 2024-04-26T04:40:06.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: overall HEALTH_OK 2024-04-26T04:40:06.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: mon.a calling monitor election 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: mon.b calling monitor election 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: mon.c calling monitor election 2024-04-26T04:40:06.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: monmap e3: 3 mons at {a=[v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0],b=[v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0],c=[v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: fsmap 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: mgrmap e13: y(active, since 45s) 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: overall HEALTH_OK 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.771 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:06.771 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:07.364 INFO:teuthology.orchestra.run.smithi111.stdout: 2024-04-26T04:40:07.364 INFO:teuthology.orchestra.run.smithi111.stdout:{"epoch":3,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","modified":"2024-04-26T04:40:01.288521Z","created":"2024-04-26T04:38:19.110602Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:3300","nonce":0},{"type":"v1","addr":"172.21.15.5:6789","nonce":0}]},"addr":"172.21.15.5:6789/0","public_addr":"172.21.15.5:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:3300","nonce":0},{"type":"v1","addr":"172.21.15.111:6789","nonce":0}]},"addr":"172.21.15.111:6789/0","public_addr":"172.21.15.111:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:3301","nonce":0},{"type":"v1","addr":"172.21.15.5:6790","nonce":0}]},"addr":"172.21.15.5:6790/0","public_addr":"172.21.15.5:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-26T04:40:07.367 INFO:teuthology.orchestra.run.smithi111.stderr:dumped monmap epoch 3 2024-04-26T04:40:07.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:07 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:07.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:07 smithi005 ceph-mon[26371]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:07.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:07 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:07 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:07 smithi005 ceph-mon[36961]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:07 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:07.747 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:07 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:07.747 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:07 smithi111 ceph-mon[30255]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:07.747 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:07 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:40:08.088 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T04:40:08.088 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T04:40:08.347 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/3014320640' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:40:08.615 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[26371]: Reconfiguring daemon mon.a on smithi005 2024-04-26T04:40:08.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/3014320640' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:08.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:08 smithi005 ceph-mon[36961]: Reconfiguring daemon mon.a on smithi005 2024-04-26T04:40:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/3014320640' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:40:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:08 smithi111 ceph-mon[30255]: Reconfiguring daemon mon.a on smithi005 2024-04-26T04:40:09.691 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:09 smithi005 ceph-mon[26371]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:09.691 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:09 smithi005 ceph-mon[36961]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:09 smithi111 ceph-mon[30255]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:11.148 INFO:teuthology.orchestra.run.smithi005.stdout:# minimal ceph.conf for b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:40:11.148 INFO:teuthology.orchestra.run.smithi005.stdout:[global] 2024-04-26T04:40:11.148 INFO:teuthology.orchestra.run.smithi005.stdout: fsid = b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:40:11.149 INFO:teuthology.orchestra.run.smithi005.stdout: mon_host = [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: Reconfiguring daemon mon.c on smithi005 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2258539033' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: Reconfiguring daemon mon.c on smithi005 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:11 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2258539033' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: Reconfiguring daemon mon.c on smithi005 2024-04-26T04:40:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:11.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:11.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:11 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2258539033' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:12.621 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T04:40:12.621 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:40:12.621 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:40:12.651 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:40:12.651 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:40:12.718 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:40:12.718 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:40:12.743 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:40:12.743 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:40:12.808 INFO:tasks.cephadm:Adding mgr.y on smithi005 2024-04-26T04:40:12.808 INFO:tasks.cephadm:Adding mgr.x on smithi111 2024-04-26T04:40:12.808 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi005=y;smithi111=x' 2024-04-26T04:40:13.096 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:40:13.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:13.463 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:13.463 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:40:13.463 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:13.463 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:13.463 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:13.463 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:13 smithi111 ceph-mon[30255]: Reconfiguring daemon mon.b on smithi111 2024-04-26T04:40:13.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[26371]: Reconfiguring daemon mon.b on smithi111 2024-04-26T04:40:13.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:40:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:40:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:40:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:13 smithi005 ceph-mon[36961]: Reconfiguring daemon mon.b on smithi111 2024-04-26T04:40:14.468 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:14.555 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.555 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.555 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:14.555 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:14.556 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:14.556 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.557 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:14.557 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.557 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.557 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:14.557 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:14.557 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:14.558 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.558 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:14.558 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.558 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:14.558 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:14.558 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:14.559 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:14.559 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:15.147 INFO:teuthology.orchestra.run.smithi111.stdout:Scheduled mgr update... 2024-04-26T04:40:15.731 DEBUG:teuthology.orchestra.run.smithi111:mgr.x> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service 2024-04-26T04:40:15.736 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T04:40:15.737 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:40:15.737 DEBUG:teuthology.orchestra.run.smithi005:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:40:15.757 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:40:15.757 DEBUG:teuthology.orchestra.run.smithi005:> stat /dev/vg_nvme/lv_1 2024-04-26T04:40:15.816 INFO:teuthology.orchestra.run.smithi005.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout:Access: 2024-04-26 04:40:00.753424858 +0000 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout:Modify: 2024-04-26 04:35:13.620731788 +0000 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout:Change: 2024-04-26 04:35:13.620731788 +0000 2024-04-26T04:40:15.817 INFO:teuthology.orchestra.run.smithi005.stdout: Birth: 2024-04-26 04:35:13.620731788 +0000 2024-04-26T04:40:15.817 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:40:15.886 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records in 2024-04-26T04:40:15.886 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records out 2024-04-26T04:40:15.886 INFO:teuthology.orchestra.run.smithi005.stderr:512 bytes copied, 0.000119203 s, 4.3 MB/s 2024-04-26T04:40:15.887 DEBUG:teuthology.orchestra.run.smithi005:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:40:15.948 DEBUG:teuthology.orchestra.run.smithi005:> stat /dev/vg_nvme/lv_2 2024-04-26T04:40:16.006 INFO:teuthology.orchestra.run.smithi005.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:40:16.006 INFO:teuthology.orchestra.run.smithi005.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout:Access: 2024-04-26 04:40:00.753424858 +0000 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout:Modify: 2024-04-26 04:35:13.946724673 +0000 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout:Change: 2024-04-26 04:35:13.946724673 +0000 2024-04-26T04:40:16.007 INFO:teuthology.orchestra.run.smithi005.stdout: Birth: 2024-04-26 04:35:13.946724673 +0000 2024-04-26T04:40:16.007 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:40:16.073 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records in 2024-04-26T04:40:16.073 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records out 2024-04-26T04:40:16.073 INFO:teuthology.orchestra.run.smithi005.stderr:512 bytes copied, 0.000124192 s, 4.1 MB/s 2024-04-26T04:40:16.075 DEBUG:teuthology.orchestra.run.smithi005:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:40:16.133 DEBUG:teuthology.orchestra.run.smithi005:> stat /dev/vg_nvme/lv_3 2024-04-26T04:40:16.189 INFO:teuthology.orchestra.run.smithi005.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout:Access: 2024-04-26 04:40:00.754424838 +0000 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout:Modify: 2024-04-26 04:35:14.281717361 +0000 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout:Change: 2024-04-26 04:35:14.281717361 +0000 2024-04-26T04:40:16.190 INFO:teuthology.orchestra.run.smithi005.stdout: Birth: 2024-04-26 04:35:14.281717361 +0000 2024-04-26T04:40:16.190 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:40:16.243 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:16.243 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi005=y;smithi111=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: Saving service mgr spec with placement smithi005=y;smithi111=x;count:2 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:40:16.244 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:16.245 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:16 smithi111 ceph-mon[30255]: Deploying daemon mgr.x on smithi111 2024-04-26T04:40:16.255 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records in 2024-04-26T04:40:16.255 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records out 2024-04-26T04:40:16.255 INFO:teuthology.orchestra.run.smithi005.stderr:512 bytes copied, 0.000122405 s, 4.2 MB/s 2024-04-26T04:40:16.256 DEBUG:teuthology.orchestra.run.smithi005:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:40:16.314 DEBUG:teuthology.orchestra.run.smithi005:> stat /dev/vg_nvme/lv_4 2024-04-26T04:40:16.371 INFO:teuthology.orchestra.run.smithi005.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout:Access: 2024-04-26 04:40:00.754424838 +0000 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout:Modify: 2024-04-26 04:35:14.628709787 +0000 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout:Change: 2024-04-26 04:35:14.628709787 +0000 2024-04-26T04:40:16.372 INFO:teuthology.orchestra.run.smithi005.stdout: Birth: 2024-04-26 04:35:14.628709787 +0000 2024-04-26T04:40:16.373 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi005=y;smithi111=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: Saving service mgr spec with placement smithi005=y;smithi111=x;count:2 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:16.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[36961]: Deploying daemon mgr.x on smithi111 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi005=y;smithi111=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: Saving service mgr spec with placement smithi005=y;smithi111=x;count:2 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:16.436 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:16.437 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:40:16.437 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:40:16.437 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:40:16.437 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:16.437 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:16 smithi005 ceph-mon[26371]: Deploying daemon mgr.x on smithi111 2024-04-26T04:40:16.437 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records in 2024-04-26T04:40:16.437 INFO:teuthology.orchestra.run.smithi005.stderr:1+0 records out 2024-04-26T04:40:16.437 INFO:teuthology.orchestra.run.smithi005.stderr:512 bytes copied, 0.000116787 s, 4.4 MB/s 2024-04-26T04:40:16.439 DEBUG:teuthology.orchestra.run.smithi005:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:40:16.497 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:40:16.497 DEBUG:teuthology.orchestra.run.smithi111:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:40:16.512 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:40:16.512 DEBUG:teuthology.orchestra.run.smithi111:> stat /dev/vg_nvme/lv_1 2024-04-26T04:40:16.567 INFO:teuthology.orchestra.run.smithi111.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:40:16.567 INFO:teuthology.orchestra.run.smithi111.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:16.567 INFO:teuthology.orchestra.run.smithi111.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-26T04:40:16.568 INFO:teuthology.orchestra.run.smithi111.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:16.568 INFO:teuthology.orchestra.run.smithi111.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:16.568 INFO:teuthology.orchestra.run.smithi111.stdout:Access: 2024-04-26 04:39:52.019831991 +0000 2024-04-26T04:40:16.568 INFO:teuthology.orchestra.run.smithi111.stdout:Modify: 2024-04-26 04:35:09.372572899 +0000 2024-04-26T04:40:16.568 INFO:teuthology.orchestra.run.smithi111.stdout:Change: 2024-04-26 04:35:09.372572899 +0000 2024-04-26T04:40:16.568 INFO:teuthology.orchestra.run.smithi111.stdout: Birth: 2024-04-26 04:35:09.372572899 +0000 2024-04-26T04:40:16.568 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:40:16.631 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records in 2024-04-26T04:40:16.631 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records out 2024-04-26T04:40:16.631 INFO:teuthology.orchestra.run.smithi111.stderr:512 bytes copied, 0.000192868 s, 2.7 MB/s 2024-04-26T04:40:16.632 DEBUG:teuthology.orchestra.run.smithi111:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:40:16.689 DEBUG:teuthology.orchestra.run.smithi111:> stat /dev/vg_nvme/lv_2 2024-04-26T04:40:16.744 INFO:teuthology.orchestra.run.smithi111.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout:Access: 2024-04-26 04:39:52.019831991 +0000 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout:Modify: 2024-04-26 04:35:09.721566400 +0000 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout:Change: 2024-04-26 04:35:09.721566400 +0000 2024-04-26T04:40:16.745 INFO:teuthology.orchestra.run.smithi111.stdout: Birth: 2024-04-26 04:35:09.721566400 +0000 2024-04-26T04:40:16.745 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:40:16.807 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records in 2024-04-26T04:40:16.807 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records out 2024-04-26T04:40:16.807 INFO:teuthology.orchestra.run.smithi111.stderr:512 bytes copied, 0.000252528 s, 2.0 MB/s 2024-04-26T04:40:16.808 DEBUG:teuthology.orchestra.run.smithi111:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:40:16.864 DEBUG:teuthology.orchestra.run.smithi111:> stat /dev/vg_nvme/lv_3 2024-04-26T04:40:16.918 INFO:teuthology.orchestra.run.smithi111.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:40:16.918 INFO:teuthology.orchestra.run.smithi111.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:16.918 INFO:teuthology.orchestra.run.smithi111.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-26T04:40:16.918 INFO:teuthology.orchestra.run.smithi111.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:16.918 INFO:teuthology.orchestra.run.smithi111.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:16.919 INFO:teuthology.orchestra.run.smithi111.stdout:Access: 2024-04-26 04:39:52.020831974 +0000 2024-04-26T04:40:16.919 INFO:teuthology.orchestra.run.smithi111.stdout:Modify: 2024-04-26 04:35:10.038560497 +0000 2024-04-26T04:40:16.919 INFO:teuthology.orchestra.run.smithi111.stdout:Change: 2024-04-26 04:35:10.038560497 +0000 2024-04-26T04:40:16.919 INFO:teuthology.orchestra.run.smithi111.stdout: Birth: 2024-04-26 04:35:10.038560497 +0000 2024-04-26T04:40:16.919 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:40:17.101 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records in 2024-04-26T04:40:17.101 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records out 2024-04-26T04:40:17.101 INFO:teuthology.orchestra.run.smithi111.stderr:512 bytes copied, 0.000197797 s, 2.6 MB/s 2024-04-26T04:40:17.103 DEBUG:teuthology.orchestra.run.smithi111:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:40:17.118 DEBUG:teuthology.orchestra.run.smithi111:> stat /dev/vg_nvme/lv_4 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout:Access: 2024-04-26 04:40:17.057404487 +0000 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout:Modify: 2024-04-26 04:35:10.363554445 +0000 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout:Change: 2024-04-26 04:35:10.363554445 +0000 2024-04-26T04:40:17.177 INFO:teuthology.orchestra.run.smithi111.stdout: Birth: 2024-04-26 04:35:10.363554445 +0000 2024-04-26T04:40:17.178 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:40:17.287 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records in 2024-04-26T04:40:17.287 INFO:teuthology.orchestra.run.smithi111.stderr:1+0 records out 2024-04-26T04:40:17.287 INFO:teuthology.orchestra.run.smithi111.stderr:512 bytes copied, 0.00022587 s, 2.3 MB/s 2024-04-26T04:40:17.288 DEBUG:teuthology.orchestra.run.smithi111:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:40:17.302 INFO:tasks.cephadm:Deploying osd.0 on smithi005 with /dev/vg_nvme/lv_4... 2024-04-26T04:40:17.303 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:40:17.514 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 systemd[1]: Starting Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:40:17.569 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:17.767 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 podman[32145]: 2024-04-26 04:40:17.416234832 +0000 UTC m=+0.017593811 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:40:17.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 podman[32145]: 2024-04-26 04:40:17.523380667 +0000 UTC m=+0.124739634 container create 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux ) 2024-04-26T04:40:17.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 podman[32145]: 2024-04-26 04:40:17.767321563 +0000 UTC m=+0.368680530 container init 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, ceph=True, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T04:40:18.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 podman[32145]: 2024-04-26 04:40:17.770040458 +0000 UTC m=+0.371399427 container start 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:40:18.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 ceph-mgr[32164]: -- 172.21.15.111:0/2695526070 <== mon.1 v2:172.21.15.111:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56397bd029c0 con 0x56397bcd8c00 2024-04-26T04:40:18.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 bash[32145]: 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 2024-04-26T04:40:18.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 systemd[1]: Started Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:18.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:17.917+0000 7f700d6b41c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:40:18.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[26371]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[36961]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:18.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:18.082+0000 7f700d6b41c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:40:18.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:18.187+0000 7f700d6b41c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:40:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-mon[30255]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:18.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:18.791 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:18.523+0000 7f700d6b41c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:40:18.791 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:18.605+0000 7f700d6b41c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:40:19.412 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:40:19.412 INFO:teuthology.orchestra.run.smithi005.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:40:19.413 INFO:teuthology.orchestra.run.smithi005.stderr: stderr: 10+0 records in 2024-04-26T04:40:19.413 INFO:teuthology.orchestra.run.smithi005.stderr:10+0 records out 2024-04-26T04:40:19.413 INFO:teuthology.orchestra.run.smithi005.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0085335 s, 1.2 GB/s 2024-04-26T04:40:19.413 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping successful for: 2024-04-26T04:40:19.990 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi005:vg_nvme/lv_4 2024-04-26T04:40:20.125 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:19.791+0000 7f700d6b41c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:40:20.125 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:19.955+0000 7f700d6b41c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:40:20.125 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:20.123+0000 7f700d6b41c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:40:20.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:40:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:20 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:20.386 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:20.386 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.387 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.387 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:20.387 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:20.387 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:20.387 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:40:20.387 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:40:20.388 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:20.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:20.385+0000 7f700d6b41c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:40:20.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:20.475+0000 7f700d6b41c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:40:20.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:20.568+0000 7f700d6b41c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:40:20.987 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:21.160 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:20.841+0000 7f700d6b41c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:40:21.414 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: Reconfiguring daemon mgr.y on smithi005 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.415 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: Reconfiguring daemon mgr.y on smithi005 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:21.300+0000 7f700d6b41c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: Reconfiguring daemon mgr.y on smithi005 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:21.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:21.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:21.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:22.091 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:21.819+0000 7f700d6b41c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:40:22.091 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:21.912+0000 7f700d6b41c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:40:22.091 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:22.089+0000 7f700d6b41c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:40:22.371 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:22 smithi005 ceph-mon[26371]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:22.371 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:22 smithi005 ceph-mon[36961]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:22.378 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:22 smithi111 ceph-mon[30255]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:22.630 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:22.376+0000 7f700d6b41c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:40:22.630 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:22.628+0000 7f700d6b41c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:40:22.994 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:22.863+0000 7f700d6b41c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:40:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:23 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:40:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:23 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:40:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:23 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:23.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:22.992+0000 7f700d6b41c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:40:23.269 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:23.083+0000 7f700d6b41c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:40:23.269 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:23.199+0000 7f700d6b41c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:40:23.435 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:23 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:40:23.435 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:23 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:40:23.435 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:23 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:23.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:23 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:40:23.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:23 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:40:23.435 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:23 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:23.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:23.289+0000 7f700d6b41c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:40:24.471 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:24 smithi005 ceph-mon[36961]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:24.472 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:24 smithi005 ceph-mon[36961]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:24.472 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:24 smithi005 ceph-mon[26371]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:24.472 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:24 smithi005 ceph-mon[26371]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:24 smithi111 ceph-mon[30255]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:24 smithi111 ceph-mon[30255]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:24.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:24 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:24.073+0000 7f700d6b41c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:40:24.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:40:24 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:40:24.263+0000 7f700d6b41c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:40:25.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[26371]: Standby manager daemon x started 2024-04-26T04:40:25.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:40:25.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:40:25.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:40:25.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:40:25.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[36961]: Standby manager daemon x started 2024-04-26T04:40:25.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:40:25.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:40:25.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:40:25.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:25 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:40:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:25 smithi111 ceph-mon[30255]: Standby manager daemon x started 2024-04-26T04:40:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:25 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:40:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:25 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:40:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:25 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:40:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:25 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/731531127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1414393126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]: dispatch 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]: dispatch 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]': finished 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2298696266' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1414393126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]: dispatch 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]: dispatch 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]': finished 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:26.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:26 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2298696266' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:40:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-26T04:40:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:40:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1414393126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]: dispatch 2024-04-26T04:40:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]: dispatch 2024-04-26T04:40:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "daa9ec57-3f2d-4901-9460-dae474808678"}]': finished 2024-04-26T04:40:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:40:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:26 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2298696266' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:40:28.377 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:28 smithi005 ceph-mon[26371]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:28.377 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:28 smithi005 ceph-mon[36961]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:28.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:28 smithi111 ceph-mon[30255]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:30.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:30 smithi005 ceph-mon[26371]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:30.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:30 smithi005 ceph-mon[36961]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:30.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:30 smithi111 ceph-mon[30255]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:32 smithi005 ceph-mon[26371]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:32.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:32 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:40:32.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:32 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:32 smithi005 ceph-mon[36961]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:32 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:40:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:32 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:32.501 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:32 smithi111 ceph-mon[30255]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:32.501 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:32 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:40:32.501 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:32 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:33.399 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:33 smithi005 ceph-mon[36961]: Deploying daemon osd.0 on smithi005 2024-04-26T04:40:33.400 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:33 smithi005 ceph-mon[26371]: Deploying daemon osd.0 on smithi005 2024-04-26T04:40:33.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:33 smithi111 ceph-mon[30255]: Deploying daemon osd.0 on smithi005 2024-04-26T04:40:34.391 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:34 smithi005 ceph-mon[26371]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:34.391 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:34 smithi005 ceph-mon[36961]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:34 smithi111 ceph-mon[30255]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:36.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:36 smithi005 ceph-mon[26371]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:36 smithi005 ceph-mon[36961]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:36 smithi111 ceph-mon[30255]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[26371]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:37.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:37.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:37.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:37.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:37 smithi005 ceph-mon[36961]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:37.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:37 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:37.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:37 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:37.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:37 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:37.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:37 smithi111 ceph-mon[30255]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:39.459 INFO:teuthology.orchestra.run.smithi005.stdout:Created osd(s) 0 on host 'smithi005' 2024-04-26T04:40:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[26371]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[26371]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[36961]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:40 smithi005 ceph-mon[36961]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:40:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:40 smithi111 ceph-mon[30255]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:40 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:40 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:40 smithi111 ceph-mon[30255]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:40:40.300 DEBUG:teuthology.orchestra.run.smithi005:osd.0> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T04:40:40.302 INFO:tasks.cephadm:Deploying osd.1 on smithi005 with /dev/vg_nvme/lv_3... 2024-04-26T04:40:40.302 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:40:40.606 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:41.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[26371]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:40:41.728 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:41.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:40:41.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:41.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-mon[36961]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:41.729 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[40912]: 2024-04-26T04:40:41.418+0000 7efe9244a700 -1 osd.0 0 waiting for initial osdmap 2024-04-26T04:40:41.729 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:40:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[40912]: 2024-04-26T04:40:41.420+0000 7efe89a3e700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:40:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:40:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:40:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:40:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:41.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:41 smithi111 ceph-mon[30255]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:42.465 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:40:42.466 INFO:teuthology.orchestra.run.smithi005.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:40:42.466 INFO:teuthology.orchestra.run.smithi005.stderr: stderr: 10+0 records in 2024-04-26T04:40:42.466 INFO:teuthology.orchestra.run.smithi005.stderr:10+0 records out 2024-04-26T04:40:42.466 INFO:teuthology.orchestra.run.smithi005.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0138322 s, 758 MB/s 2024-04-26T04:40:42.466 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping successful for: 2024-04-26T04:40:42.517 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[26371]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[26371]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[36961]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[36961]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:42.518 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:42 smithi111 ceph-mon[30255]: from='osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:40:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:42 smithi111 ceph-mon[30255]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:40:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.404 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi005:vg_nvme/lv_3 2024-04-26T04:40:43.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:40:43.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:40:43.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172] boot 2024-04-26T04:40:43.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[26371]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172] boot 2024-04-26T04:40:43.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:40:43.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:43 smithi005 ceph-mon[36961]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:43.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:40:43.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:40:43.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: osd.0 [v2:172.21.15.5:6802/1943995172,v1:172.21.15.5:6803/1943995172] boot 2024-04-26T04:40:43.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:40:43.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:40:43.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:43 smithi111 ceph-mon[30255]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:40:43.906 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:44.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:44 smithi005 ceph-mon[26371]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:40:44.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:44 smithi005 ceph-mon[36961]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:40:44.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:44 smithi111 ceph-mon[30255]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:40:45.676 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:45 smithi005 ceph-mon[26371]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:45.677 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:45 smithi005 ceph-mon[36961]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:45.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:45 smithi111 ceph-mon[30255]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:46.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi005 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi005 to 16069M 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi005 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi005 to 16069M 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:46.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:46.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi005 2024-04-26T04:40:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:40:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi005 to 16069M 2024-04-26T04:40:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:47.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:40:47.896 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[26371]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:47.896 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:40:47.896 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:40:47.896 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[26371]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:47.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[36961]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:47.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:40:47.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:40:47.898 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.898 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:47 smithi005 ceph-mon[36961]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:47 smithi111 ceph-mon[30255]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:47 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:40:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:47 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:40:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:47 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:47 smithi111 ceph-mon[30255]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3752063237' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]: dispatch 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]: dispatch 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]': finished 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:40:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:50 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3372373916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3752063237' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]: dispatch 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]: dispatch 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]': finished 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:40:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3372373916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3752063237' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]: dispatch 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]: dispatch 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa"}]': finished 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:40:50.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:50 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3372373916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:40:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:52 smithi111 ceph-mon[30255]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:52.446 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:52 smithi005 ceph-mon[26371]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:52.447 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:52 smithi005 ceph-mon[36961]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:54 smithi005 ceph-mon[26371]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:54 smithi005 ceph-mon[36961]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:54 smithi111 ceph-mon[30255]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:56 smithi111 ceph-mon[30255]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:56 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:40:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:56 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:56.278 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:56 smithi005 ceph-mon[26371]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:56.278 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:56 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:40:56.278 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:56 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:56.279 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:56 smithi005 ceph-mon[36961]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:56.279 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:56 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:40:56.279 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:56 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:57 smithi005 ceph-mon[26371]: Deploying daemon osd.1 on smithi005 2024-04-26T04:40:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:57 smithi005 ceph-mon[36961]: Deploying daemon osd.1 on smithi005 2024-04-26T04:40:57.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:57 smithi111 ceph-mon[30255]: Deploying daemon osd.1 on smithi005 2024-04-26T04:40:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:40:58 smithi005 ceph-mon[26371]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:40:58 smithi005 ceph-mon[36961]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:40:58.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:40:58 smithi111 ceph-mon[30255]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:00.096 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:00 smithi005 ceph-mon[26371]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:00.096 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:00 smithi005 ceph-mon[36961]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:00.320 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:00 smithi111 ceph-mon[30255]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[26371]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:01 smithi005 ceph-mon[36961]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:01 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:01 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:01 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:01 smithi111 ceph-mon[30255]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:04.157 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:04.157 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[36961]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.158 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:04.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:04.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:41:04.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:04 smithi005 ceph-mon[26371]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:41:04.225 INFO:teuthology.orchestra.run.smithi005.stdout:Created osd(s) 1 on host 'smithi005' 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:41:04.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:04 smithi111 ceph-mon[30255]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:41:04.794 DEBUG:teuthology.orchestra.run.smithi005:osd.1> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T04:41:04.796 INFO:tasks.cephadm:Deploying osd.2 on smithi005 with /dev/vg_nvme/lv_2... 2024-04-26T04:41:04.796 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[26371]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:05.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-mon[36961]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:05.482 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[46613]: 2024-04-26T04:41:05.325+0000 7face2b26700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T04:41:05.482 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[46613]: 2024-04-26T04:41:05.328+0000 7facdf925700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:05 smithi111 ceph-mon[30255]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:06.017 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:06.219 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[26371]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:06.219 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[26371]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:06.219 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[26371]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:41:06.219 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:06.219 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:06.220 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[36961]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:06.220 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[36961]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:06.220 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[36961]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:41:06.220 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:06.220 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:06 smithi111 ceph-mon[30255]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:41:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:06 smithi111 ceph-mon[30255]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:06 smithi111 ceph-mon[30255]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:41:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:41:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:41:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[26371]: osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947] boot 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[26371]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[26371]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[36961]: osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947] boot 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[36961]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:07 smithi005 ceph-mon[36961]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:07 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:41:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:07 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:41:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:07 smithi111 ceph-mon[30255]: osd.1 [v2:172.21.15.5:6810/2192671947,v1:172.21.15.5:6811/2192671947] boot 2024-04-26T04:41:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:07 smithi111 ceph-mon[30255]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:41:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:07 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:41:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:07 smithi111 ceph-mon[30255]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi005 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi005 to 8034M 2024-04-26T04:41:08.627 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi005 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi005 to 8034M 2024-04-26T04:41:08.628 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.629 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:08.629 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:08.629 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:08 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:41:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi005 2024-04-26T04:41:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:08.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi005 to 8034M 2024-04-26T04:41:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:08 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:09.000 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:41:09.000 INFO:teuthology.orchestra.run.smithi005.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:41:09.000 INFO:teuthology.orchestra.run.smithi005.stderr: stderr: 10+0 records in 2024-04-26T04:41:09.000 INFO:teuthology.orchestra.run.smithi005.stderr:10+0 records out 2024-04-26T04:41:09.000 INFO:teuthology.orchestra.run.smithi005.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00842801 s, 1.2 GB/s 2024-04-26T04:41:09.001 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping successful for: 2024-04-26T04:41:09.538 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:09 smithi005 ceph-mon[26371]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:09.538 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:09 smithi005 ceph-mon[36961]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:09.566 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi005:vg_nvme/lv_2 2024-04-26T04:41:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:09 smithi111 ceph-mon[30255]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:09.828 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[26371]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[36961]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:41:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:12 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:12 smithi111 ceph-mon[30255]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:12 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:41:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:12 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:41:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:12 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:13 smithi111 ceph-mon[30255]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:13.332 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:13 smithi005 ceph-mon[26371]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:13.332 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:13 smithi005 ceph-mon[36961]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:14.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[26371]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3669505404' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d89110dd-b5fa-45e6-a733-76e6b536be58"}]: dispatch 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3669505404' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d89110dd-b5fa-45e6-a733-76e6b536be58"}]': finished 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[26371]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[36961]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3669505404' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d89110dd-b5fa-45e6-a733-76e6b536be58"}]: dispatch 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3669505404' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d89110dd-b5fa-45e6-a733-76e6b536be58"}]': finished 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[36961]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:41:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:14 smithi111 ceph-mon[30255]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:14 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3669505404' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d89110dd-b5fa-45e6-a733-76e6b536be58"}]: dispatch 2024-04-26T04:41:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:14 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3669505404' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d89110dd-b5fa-45e6-a733-76e6b536be58"}]': finished 2024-04-26T04:41:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:14 smithi111 ceph-mon[30255]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:41:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:15.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:15 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/318207921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:41:15.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:15 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/318207921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:41:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:15 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/318207921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:41:16.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:16 smithi005 ceph-mon[26371]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:16.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:16 smithi005 ceph-mon[36961]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:16 smithi111 ceph-mon[30255]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:18 smithi005 ceph-mon[26371]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:18 smithi005 ceph-mon[36961]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:18 smithi111 ceph-mon[30255]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:20 smithi005 ceph-mon[26371]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:20 smithi005 ceph-mon[36961]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:20 smithi111 ceph-mon[30255]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:41:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:41:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:41:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:22.232 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:22 smithi005 ceph-mon[26371]: Deploying daemon osd.2 on smithi005 2024-04-26T04:41:22.671 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:22 smithi005 ceph-mon[26371]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:22.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:22 smithi005 ceph-mon[36961]: Deploying daemon osd.2 on smithi005 2024-04-26T04:41:22.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:22 smithi005 ceph-mon[36961]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:22.672 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:22 smithi111 ceph-mon[30255]: Deploying daemon osd.2 on smithi005 2024-04-26T04:41:22.672 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:22 smithi111 ceph-mon[30255]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:24.170 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:24 smithi005 ceph-mon[26371]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:24.171 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:24 smithi005 ceph-mon[36961]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:24 smithi111 ceph-mon[30255]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:26.467 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[26371]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:26.467 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:26.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:26.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:26.468 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[36961]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:26.468 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:26.468 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:26.468 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:26 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:26 smithi111 ceph-mon[30255]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:26 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:26 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:26 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:28.416 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:28 smithi005 ceph-mon[26371]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:28.416 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:28 smithi005 ceph-mon[36961]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:28.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:28 smithi111 ceph-mon[30255]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:28.705 INFO:teuthology.orchestra.run.smithi005.stdout:Created osd(s) 2 on host 'smithi005' 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.315 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.316 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.317 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:29 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:29.341 DEBUG:teuthology.orchestra.run.smithi005:osd.2> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T04:41:29.343 INFO:tasks.cephadm:Deploying osd.3 on smithi005 with /dev/vg_nvme/lv_1... 2024-04-26T04:41:29.343 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:41:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:41:29.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:41:29.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:29.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:29 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[26371]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[26371]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:41:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[26371]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:41:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[26371]: from='osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[26371]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[36961]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[36961]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:41:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[36961]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:41:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[36961]: from='osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-mon[36961]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:30.231 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[52705]: 2024-04-26T04:41:30.080+0000 7f2570010700 -1 osd.2 0 waiting for initial osdmap 2024-04-26T04:41:30.231 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[52705]: 2024-04-26T04:41:30.083+0000 7f256b60c700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:41:30.419 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:30 smithi111 ceph-mon[30255]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:30.419 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:30 smithi111 ceph-mon[30255]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:41:30.419 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:30 smithi111 ceph-mon[30255]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:41:30.420 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:30.420 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:30 smithi111 ceph-mon[30255]: from='osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:30.420 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:30 smithi111 ceph-mon[30255]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:30.561 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[26371]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[26371]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[36961]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[36961]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.369 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:31 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:31 smithi111 ceph-mon[30255]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:31 smithi111 ceph-mon[30255]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:41:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:31 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:31 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:31 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:32.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:41:32.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:41:32.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[26371]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[26371]: osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348] boot 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[26371]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[36961]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[36961]: osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348] boot 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[36961]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:41:32.160 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:32 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:32 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:41:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:32 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:41:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:32 smithi111 ceph-mon[30255]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:41:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:32 smithi111 ceph-mon[30255]: osd.2 [v2:172.21.15.5:6818/1951395348,v1:172.21.15.5:6819/1951395348] boot 2024-04-26T04:41:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:32 smithi111 ceph-mon[30255]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:41:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:32 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:41:32.992 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:41:32.992 INFO:teuthology.orchestra.run.smithi005.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:41:32.992 INFO:teuthology.orchestra.run.smithi005.stderr: stderr: 10+0 records in 2024-04-26T04:41:32.992 INFO:teuthology.orchestra.run.smithi005.stderr:10+0 records out 2024-04-26T04:41:32.992 INFO:teuthology.orchestra.run.smithi005.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00963683 s, 1.1 GB/s 2024-04-26T04:41:32.992 INFO:teuthology.orchestra.run.smithi005.stderr:--> Zapping successful for: 2024-04-26T04:41:33.590 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:33 smithi005 ceph-mon[26371]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:41:33.590 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:33 smithi005 ceph-mon[36961]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:41:33.618 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi005:vg_nvme/lv_1 2024-04-26T04:41:33.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:33 smithi111 ceph-mon[30255]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:41:34.065 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:41:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:41:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:41:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi005 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi005 to 5356M 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi005 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi005 to 5356M 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:34.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:34.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi005 2024-04-26T04:41:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi005 to 5356M 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:34.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:35.376 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56901]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:41:35.376 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56901]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:35.376 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56901]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:35.376 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56901]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:35.671 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:35 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:41:35.671 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:35 smithi005 ceph-mon[26371]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:41:35.671 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:35 smithi005 ceph-mon[26371]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:35.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:35 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:41:35.672 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:35 smithi005 ceph-mon[36961]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:41:35.672 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:35 smithi005 ceph-mon[36961]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:35.672 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56928]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:41:35.672 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56928]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:35.672 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56928]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:35.672 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:41:35 smithi005 sudo[56928]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:35.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:35 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:41:35.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:35 smithi111 ceph-mon[30255]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:41:35.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:35 smithi111 ceph-mon[30255]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:36.303 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[56956]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:41:36.303 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[56956]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:36.303 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[56956]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:36.303 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[56956]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:36.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:41:36.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:41:36.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:36.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:36.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:41:36.605 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:41:36.606 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:36.606 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:41:36.606 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:41:36.606 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:36.606 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:36.607 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:41:36.607 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:41:36.607 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:36 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:36.607 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57085]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:41:36.607 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57085]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:36.607 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57085]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:36.608 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57085]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi005:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:41:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:36 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:37.230 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57107]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:41:37.230 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57107]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:37.230 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57107]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:37.230 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:36 smithi005 sudo[57107]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:37.581 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57116]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:41:37.581 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57116]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:37.581 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57116]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:37.581 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57116]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57145]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:41:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57145]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57145]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:38.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:37 smithi005 sudo[57145]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:38.597 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 ceph-mon[30255]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:38.597 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32866]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:41:38.597 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32866]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:38.597 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32866]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:38.597 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32866]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:38.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:38 smithi005 ceph-mon[36961]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:38 smithi005 sudo[57191]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:41:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:38 smithi005 sudo[57191]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:38 smithi005 sudo[57191]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:38 smithi005 sudo[57191]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:38 smithi005 ceph-mon[26371]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32869]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:41:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32869]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32869]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:38 smithi111 sudo[32869]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:39.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57227]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:41:39.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57227]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:39.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57227]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:39.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57227]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3815254494' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]': finished 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:40.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:40.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:39 smithi111 ceph-mon[30255]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57230]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:41:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57230]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:41:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57230]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:41:40.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 sudo[57230]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:41:40.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:40.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:40.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:40.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3815254494' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]: dispatch 2024-04-26T04:41:40.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]: dispatch 2024-04-26T04:41:40.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:40.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]': finished 2024-04-26T04:41:40.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:41:40.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:40.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:40.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[36961]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:40.234 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:40.235 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:40.235 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.235 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.235 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.235 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:40.236 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.236 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.236 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.236 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3815254494' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]: dispatch 2024-04-26T04:41:40.236 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]: dispatch 2024-04-26T04:41:40.236 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:40.237 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06dcdf4a-cd71-407e-b50d-5a6ebc5535ee"}]': finished 2024-04-26T04:41:40.237 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:41:40.237 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:40.237 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:40.237 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:41:40.238 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:41:40.238 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:41:40.238 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:39 smithi005 ceph-mon[26371]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:40 smithi111 ceph-mon[30255]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:40 smithi111 ceph-mon[30255]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:40 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2636373665' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:41:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:40 smithi005 ceph-mon[26371]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:41.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:40 smithi005 ceph-mon[26371]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:41.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:40 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2636373665' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:41:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:40 smithi005 ceph-mon[36961]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:41:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:40 smithi005 ceph-mon[36961]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:41:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:40 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2636373665' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:41:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:41 smithi111 ceph-mon[30255]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:42.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:41 smithi005 ceph-mon[26371]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:42.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:41 smithi005 ceph-mon[36961]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:44 smithi111 ceph-mon[30255]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:41:44.310 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:44 smithi005 ceph-mon[36961]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:41:44.311 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:44 smithi005 ceph-mon[26371]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:41:46.293 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:46 smithi005 ceph-mon[26371]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:41:46.293 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:41:46.293 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:46 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:46.294 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:46 smithi005 ceph-mon[36961]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:41:46.294 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:41:46.294 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:46 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:46 smithi111 ceph-mon[30255]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:41:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:41:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:46 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:47.333 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:47 smithi005 ceph-mon[26371]: Deploying daemon osd.3 on smithi005 2024-04-26T04:41:47.333 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:47 smithi005 ceph-mon[36961]: Deploying daemon osd.3 on smithi005 2024-04-26T04:41:47.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:47 smithi111 ceph-mon[30255]: Deploying daemon osd.3 on smithi005 2024-04-26T04:41:48.264 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:48 smithi005 ceph-mon[26371]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:41:48.264 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:48 smithi005 ceph-mon[36961]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:41:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:48 smithi111 ceph-mon[30255]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:41:50.475 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:50 smithi005 ceph-mon[26371]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-04-26T04:41:50.475 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:50 smithi005 ceph-mon[36961]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-04-26T04:41:50.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:50 smithi111 ceph-mon[30255]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-04-26T04:41:52.176 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[26371]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:52.177 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:51 smithi005 ceph-mon[36961]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:41:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:51 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:51 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:51 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:51 smithi111 ceph-mon[30255]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[36961]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[36961]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[26371]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:54.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:54.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:54 smithi005 ceph-mon[26371]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:41:54.436 INFO:teuthology.orchestra.run.smithi005.stdout:Created osd(s) 3 on host 'smithi005' 2024-04-26T04:41:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:54 smithi111 ceph-mon[30255]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:41:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:54 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:54 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:54 smithi111 ceph-mon[30255]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:41:55.069 DEBUG:teuthology.orchestra.run.smithi005:osd.3> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T04:41:55.072 INFO:tasks.cephadm:Deploying osd.4 on smithi111 with /dev/vg_nvme/lv_4... 2024-04-26T04:41:55.072 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:41:55.310 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:55.310 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:55.310 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.310 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.311 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.311 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:55.311 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:41:55.311 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:41:55.311 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:55.311 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[36961]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:41:55.312 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:55.313 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-mon[26371]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:55.325 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:55.447 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:41:55.448 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:41:55.448 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:55.448 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:55 smithi111 ceph-mon[30255]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:41:55.959 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[58888]: 2024-04-26T04:41:55.680+0000 7f56f4620700 -1 osd.3 0 waiting for initial osdmap 2024-04-26T04:41:55.959 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:41:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[58888]: 2024-04-26T04:41:55.684+0000 7f56f141f700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:41:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[26371]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[26371]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[26371]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:41:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:56.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[36961]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:56.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[36961]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:56.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[36961]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:41:56.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:56.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:56 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:56.344 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:56 smithi111 ceph-mon[30255]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:41:56.344 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:56 smithi111 ceph-mon[30255]: from='osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]': finished 2024-04-26T04:41:56.344 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:56 smithi111 ceph-mon[30255]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:41:56.344 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:56 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:56.344 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:56 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:57.144 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:41:57.144 INFO:teuthology.orchestra.run.smithi111.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:41:57.144 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10+0 records in 2024-04-26T04:41:57.144 INFO:teuthology.orchestra.run.smithi111.stderr:10+0 records out 2024-04-26T04:41:57.145 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129106 s, 812 MB/s 2024-04-26T04:41:57.145 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping successful for: 2024-04-26T04:41:57.750 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi111:vg_nvme/lv_4 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[36961]: osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980] boot 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[36961]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[36961]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:41:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:41:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[26371]: osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980] boot 2024-04-26T04:41:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[26371]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:41:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:57 smithi005 ceph-mon[26371]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:41:58.000 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:41:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:57 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:41:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:57 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:41:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:57 smithi111 ceph-mon[30255]: osd.3 [v2:172.21.15.5:6826/1448676980,v1:172.21.15.5:6827/1448676980] boot 2024-04-26T04:41:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:57 smithi111 ceph-mon[30255]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:41:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:41:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:57 smithi111 ceph-mon[30255]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi005 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi005 to 4017M 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi005 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi005 to 4017M 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:58.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:58 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:41:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi005 2024-04-26T04:41:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi005 to 4017M 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:58 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:41:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:41:59 smithi005 ceph-mon[26371]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:41:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:41:59 smithi005 ceph-mon[36961]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:41:59 smithi111 ceph-mon[30255]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:00.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:00 smithi111 ceph-mon[30255]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:00.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:00 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:00.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:00 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:00.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:00 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:00.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[26371]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[36961]: from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:00 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:01 smithi005 ceph-mon[26371]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:01 smithi005 ceph-mon[36961]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:01 smithi111 ceph-mon[30255]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/1489017533' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]: dispatch 2024-04-26T04:42:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]: dispatch 2024-04-26T04:42:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]': finished 2024-04-26T04:42:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[26371]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:42:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:02.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/1489017533' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]: dispatch 2024-04-26T04:42:02.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]: dispatch 2024-04-26T04:42:02.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]': finished 2024-04-26T04:42:02.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[36961]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:42:02.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:02 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:02 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/1489017533' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]: dispatch 2024-04-26T04:42:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:02 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]: dispatch 2024-04-26T04:42:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:02 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fab3e60-abe9-4e0f-812c-4333ab56419c"}]': finished 2024-04-26T04:42:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:02 smithi111 ceph-mon[30255]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:42:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:02 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:03 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/877328009' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:03 smithi005 ceph-mon[26371]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:03 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/877328009' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:03 smithi005 ceph-mon[36961]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:03 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/877328009' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:03 smithi111 ceph-mon[30255]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:06 smithi111 ceph-mon[30255]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:06 smithi005 ceph-mon[26371]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:06 smithi005 ceph-mon[36961]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:08 smithi005 ceph-mon[26371]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:08 smithi005 ceph-mon[36961]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:08.495 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:08 smithi111 ceph-mon[30255]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:09.391 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:42:09.392 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:09.392 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:09 smithi111 ceph-mon[30255]: Deploying daemon osd.4 on smithi111 2024-04-26T04:42:09.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:42:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:09 smithi005 ceph-mon[26371]: Deploying daemon osd.4 on smithi111 2024-04-26T04:42:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:42:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:09 smithi005 ceph-mon[36961]: Deploying daemon osd.4 on smithi111 2024-04-26T04:42:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:10 smithi111 ceph-mon[30255]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:10.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:10 smithi005 ceph-mon[26371]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:10 smithi005 ceph-mon[36961]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:12.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:12 smithi111 ceph-mon[30255]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:12.282 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:12 smithi005 ceph-mon[26371]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:12.282 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:12 smithi005 ceph-mon[36961]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[26371]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[36961]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:14 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:14.500 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:14 smithi111 ceph-mon[30255]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:14.500 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:14.500 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:14.501 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:14 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:16.330 INFO:teuthology.orchestra.run.smithi111.stdout:Created osd(s) 4 on host 'smithi111' 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[26371]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:42:16.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:16 smithi005 ceph-mon[36961]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:42:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:16 smithi111 ceph-mon[30255]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:42:16.926 DEBUG:teuthology.orchestra.run.smithi111:osd.4> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T04:42:16.928 INFO:tasks.cephadm:Deploying osd.5 on smithi111 with /dev/vg_nvme/lv_3... 2024-04-26T04:42:16.929 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:17.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:17.454 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T04:42:17.196+0000 7fe1ece7e700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T04:42:17.454 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 04:42:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T04:42:17.200+0000 7fe1e5c75700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:42:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:42:17.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:17.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:17.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:17.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:17.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:17.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:17 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:18.298 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:42:18.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[36961]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[36961]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[36961]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[26371]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[26371]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[26371]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:42:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:18.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:18 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:18 smithi111 ceph-mon[30255]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:18 smithi111 ceph-mon[30255]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:42:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:18 smithi111 ceph-mon[30255]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:42:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:18 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[26371]: osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462] boot 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[26371]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:42:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[36961]: osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462] boot 2024-04-26T04:42:19.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[36961]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:42:19.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:19 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:19 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:42:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:19 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:42:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:19 smithi111 ceph-mon[30255]: osd.4 [v2:172.21.15.111:6800/2939693462,v1:172.21.15.111:6801/2939693462] boot 2024-04-26T04:42:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:19 smithi111 ceph-mon[30255]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:42:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:19 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:42:20.399 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:20 smithi111 ceph-mon[30255]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:20.399 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:20 smithi111 ceph-mon[30255]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:42:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:20 smithi005 ceph-mon[26371]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:20.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:20 smithi005 ceph-mon[26371]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:42:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:20 smithi005 ceph-mon[36961]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:42:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:20 smithi005 ceph-mon[36961]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:42:21.267 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:42:21.267 INFO:teuthology.orchestra.run.smithi111.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:42:21.267 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10+0 records in 2024-04-26T04:42:21.267 INFO:teuthology.orchestra.run.smithi111.stderr:10+0 records out 2024-04-26T04:42:21.267 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0174196 s, 602 MB/s 2024-04-26T04:42:21.267 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping successful for: 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:21 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:42:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:21.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:21.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:21 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:21.870 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi111:vg_nvme/lv_3 2024-04-26T04:42:22.117 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:42:22.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi111 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi111 to 17088M 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[26371]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[26371]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi111 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi111 to 17088M 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[36961]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:22 smithi005 ceph-mon[36961]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:42:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:22 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi111 2024-04-26T04:42:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:22 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi111 to 17088M 2024-04-26T04:42:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:22 smithi111 ceph-mon[30255]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:22 smithi111 ceph-mon[30255]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:42:23.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:23 smithi111 ceph-mon[30255]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:42:23.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:23 smithi005 ceph-mon[26371]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:42:23.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:23 smithi005 ceph-mon[36961]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:42:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[26371]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:42:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[36961]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:42:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:24 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:24 smithi111 ceph-mon[30255]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:42:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:24 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:24 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:24 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:25 smithi111 ceph-mon[30255]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:25 smithi005 ceph-mon[26371]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:25 smithi005 ceph-mon[36961]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:26 smithi111 ceph-mon[30255]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-04-26T04:42:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:26 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/3104468396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]: dispatch 2024-04-26T04:42:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:26 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]: dispatch 2024-04-26T04:42:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:26 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]': finished 2024-04-26T04:42:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:26 smithi111 ceph-mon[30255]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:42:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:26 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:26.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[26371]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/3104468396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]: dispatch 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]: dispatch 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]': finished 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[26371]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[36961]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/3104468396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]: dispatch 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]: dispatch 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "888dcd16-78cf-4776-a4e3-4375e0c10e53"}]': finished 2024-04-26T04:42:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[36961]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:42:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:26 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:27 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/3736941786' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:27 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/3736941786' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:27 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/3736941786' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:28.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:28 smithi111 ceph-mon[30255]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:42:28.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:28 smithi005 ceph-mon[26371]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:42:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:28 smithi005 ceph-mon[36961]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:42:29.310 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:29 smithi111 ceph-mon[30255]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-26T04:42:29.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:29 smithi005 ceph-mon[26371]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-26T04:42:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:29 smithi005 ceph-mon[36961]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-26T04:42:32.292 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:32 smithi111 ceph-mon[30255]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:32 smithi005 ceph-mon[26371]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:32 smithi005 ceph-mon[36961]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:33.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:33 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:42:33.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:33 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:33.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:33 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:42:33.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:33 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:33.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:33 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:42:33.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:33 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:34.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:34 smithi111 ceph-mon[30255]: Deploying daemon osd.5 on smithi111 2024-04-26T04:42:34.284 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:34 smithi111 ceph-mon[30255]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:34.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:34 smithi005 ceph-mon[26371]: Deploying daemon osd.5 on smithi111 2024-04-26T04:42:34.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:34 smithi005 ceph-mon[26371]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:34.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:34 smithi005 ceph-mon[36961]: Deploying daemon osd.5 on smithi111 2024-04-26T04:42:34.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:34 smithi005 ceph-mon[36961]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:36 smithi111 ceph-mon[30255]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:36.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:36 smithi005 ceph-mon[26371]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:36 smithi005 ceph-mon[36961]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:38.202 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:38 smithi111 ceph-mon[30255]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:38.202 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:38 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:38.202 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:38 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:38.202 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:38 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:38.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[26371]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[36961]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:38 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:40 smithi111 ceph-mon[30255]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:40.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:40 smithi005 ceph-mon[26371]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:40 smithi005 ceph-mon[36961]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:40.712 INFO:teuthology.orchestra.run.smithi111.stdout:Created osd(s) 5 on host 'smithi111' 2024-04-26T04:42:41.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:41 smithi111 ceph-mon[30255]: from='osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:42:41.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:41 smithi111 ceph-mon[30255]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:42:41.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.294 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.295 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:41 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.317 DEBUG:teuthology.orchestra.run.smithi111:osd.5> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T04:42:41.319 INFO:tasks.cephadm:Deploying osd.6 on smithi111 with /dev/vg_nvme/lv_2... 2024-04-26T04:42:41.319 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:42:41.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[26371]: from='osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[26371]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[36961]: from='osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[36961]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:41 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:42.380 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[36961]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[36961]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[36961]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[36961]: from='osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[36961]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[26371]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[26371]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[26371]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[26371]: from='osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:42.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:42 smithi005 ceph-mon[26371]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:42.451 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-mon[30255]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:42.451 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-mon[30255]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:42:42.451 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-mon[30255]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:42:42.451 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:42.451 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-mon[30255]: from='osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:42.451 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-mon[30255]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:42:42.452 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T04:42:42.053+0000 7f63e69ff700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T04:42:42.452 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 04:42:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T04:42:42.059+0000 7f63e37fe700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:42:42.957 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:42:43.310 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:43 smithi111 ceph-mon[30255]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:42:43.310 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:43 smithi111 ceph-mon[30255]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:42:43.310 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:43 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:43.310 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:43 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[36961]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[36961]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[26371]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[26371]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:43 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:44.212 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:44 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:42:44.212 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:44 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:42:44.212 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:44 smithi111 ceph-mon[30255]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:44.212 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:44 smithi111 ceph-mon[30255]: osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988] boot 2024-04-26T04:42:44.212 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:44 smithi111 ceph-mon[30255]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:42:44.212 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:44 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[26371]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[26371]: osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988] boot 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[26371]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:42:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[36961]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:42:44.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[36961]: osd.5 [v2:172.21.15.111:6808/251519988,v1:172.21.15.111:6809/251519988] boot 2024-04-26T04:42:44.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[36961]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:42:44.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:44 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:42:45.118 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:42:45.118 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.118 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:45.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:45.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:45.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:45.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:42:45.554 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:42:45.554 INFO:teuthology.orchestra.run.smithi111.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:42:45.554 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10+0 records in 2024-04-26T04:42:45.554 INFO:teuthology.orchestra.run.smithi111.stderr:10+0 records out 2024-04-26T04:42:45.554 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0138167 s, 759 MB/s 2024-04-26T04:42:45.555 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping successful for: 2024-04-26T04:42:46.197 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:46 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi111 2024-04-26T04:42:46.197 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:46 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi111 to 8544M 2024-04-26T04:42:46.197 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:46 smithi111 ceph-mon[30255]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:46.197 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:46 smithi111 ceph-mon[30255]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:42:46.197 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:46 smithi111 ceph-mon[30255]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:42:46.223 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi111:vg_nvme/lv_2 2024-04-26T04:42:46.473 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:42:46.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi111 2024-04-26T04:42:46.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi111 to 8544M 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[26371]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[26371]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[26371]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi111 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi111 to 8544M 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[36961]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[36961]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:42:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:46 smithi005 ceph-mon[36961]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:42:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:47 smithi111 ceph-mon[30255]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:42:47.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:47 smithi005 ceph-mon[26371]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:42:47.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:47 smithi005 ceph-mon[36961]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:42:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:48 smithi111 ceph-mon[30255]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:48 smithi111 ceph-mon[30255]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:42:48.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:48 smithi005 ceph-mon[26371]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:48.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:48 smithi005 ceph-mon[26371]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:42:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:48 smithi005 ceph-mon[36961]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:48 smithi005 ceph-mon[36961]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:42:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:49 smithi111 ceph-mon[30255]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:49 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:49 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:49.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:49 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:49.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[26371]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[36961]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:42:49.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:49 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:50 smithi111 ceph-mon[30255]: pgmap v137: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:50.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:50 smithi005 ceph-mon[26371]: pgmap v137: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:50 smithi005 ceph-mon[36961]: pgmap v137: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:42:51.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/1577837547' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]: dispatch 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]: dispatch 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]': finished 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[26371]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/1577837547' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]: dispatch 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]: dispatch 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]': finished 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[36961]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:42:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:51 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:42:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:51 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/1577837547' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]: dispatch 2024-04-26T04:42:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:51 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]: dispatch 2024-04-26T04:42:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:51 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0cdf3d72-f096-41f7-bbdb-c97483f0756d"}]': finished 2024-04-26T04:42:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:51 smithi111 ceph-mon[30255]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:42:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:51 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:42:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[26371]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-26T04:42:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/2706625528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[26371]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:42:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:42:52.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[36961]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-26T04:42:52.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/2706625528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:52.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[36961]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:42:52.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:52 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:42:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:52 smithi111 ceph-mon[30255]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-26T04:42:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:52 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/2706625528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:42:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:52 smithi111 ceph-mon[30255]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:42:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:52 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:42:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:54 smithi111 ceph-mon[30255]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-26T04:42:54.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:54 smithi005 ceph-mon[26371]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-26T04:42:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:54 smithi005 ceph-mon[36961]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-26T04:42:56.391 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:56 smithi111 ceph-mon[30255]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:42:56.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:56 smithi005 ceph-mon[26371]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:42:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:56 smithi005 ceph-mon[36961]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:42:57.342 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:42:57.342 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:57.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:42:57.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:57.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:42:57.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:58.418 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:58 smithi111 ceph-mon[30255]: Deploying daemon osd.6 on smithi111 2024-04-26T04:42:58.418 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:42:58 smithi111 ceph-mon[30255]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:42:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:58 smithi005 ceph-mon[26371]: Deploying daemon osd.6 on smithi111 2024-04-26T04:42:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:42:58 smithi005 ceph-mon[26371]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:42:58.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:58 smithi005 ceph-mon[36961]: Deploying daemon osd.6 on smithi111 2024-04-26T04:42:58.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:42:58 smithi005 ceph-mon[36961]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:00.260 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:00 smithi111 ceph-mon[30255]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:00.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:00 smithi005 ceph-mon[26371]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:00 smithi005 ceph-mon[36961]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:02.470 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:02 smithi111 ceph-mon[30255]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:02.470 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:02 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:02.470 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:02 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:02.470 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:02 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[26371]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:02.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:02.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[36961]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:02 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:04.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:04 smithi005 ceph-mon[26371]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:04.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:04 smithi005 ceph-mon[36961]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:04.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:04 smithi111 ceph-mon[30255]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:05.298 INFO:teuthology.orchestra.run.smithi111.stdout:Created osd(s) 6 on host 'smithi111' 2024-04-26T04:43:05.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:43:05.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:43:05.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:05.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:05.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:05.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:05 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:05.893 DEBUG:teuthology.orchestra.run.smithi111:osd.6> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T04:43:05.896 INFO:tasks.cephadm:Deploying osd.7 on smithi111 with /dev/vg_nvme/lv_1... 2024-04-26T04:43:05.896 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:05 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:06.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-mon[30255]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:43:06.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-mon[30255]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:43:06.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:06.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-mon[30255]: from='osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:06.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-mon[30255]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:06.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T04:43:06.498+0000 7fe0a0cd1700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T04:43:06.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 04:43:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T04:43:06.504+0000 7fe09a2c9700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:43:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[26371]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[26371]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[26371]: from='osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[26371]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[36961]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[36961]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[36961]: from='osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:06 smithi005 ceph-mon[36961]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:07.156 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:43:07.699 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:43:07.699 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:43:07.699 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.699 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.699 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235] boot 2024-04-26T04:43:07.700 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:43:07.700 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.700 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:07 smithi111 ceph-mon[30255]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:07.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235] boot 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[26371]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:43:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: osd.6 [v2:172.21.15.111:6816/1634521235,v1:172.21.15.111:6817/1634521235] boot 2024-04-26T04:43:07.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:43:07.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:07.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:07 smithi005 ceph-mon[36961]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:43:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:08 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:43:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:08 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:43:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:08 smithi005 ceph-mon[36961]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:43:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:08 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:43:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:08 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:43:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:08 smithi005 ceph-mon[26371]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:43:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:08 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:43:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:08 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:43:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:08 smithi111 ceph-mon[30255]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:43:10.025 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:43:10.025 INFO:teuthology.orchestra.run.smithi111.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:43:10.025 INFO:teuthology.orchestra.run.smithi111.stderr: stderr: 10+0 records in 2024-04-26T04:43:10.025 INFO:teuthology.orchestra.run.smithi111.stderr:10+0 records out 2024-04-26T04:43:10.025 INFO:teuthology.orchestra.run.smithi111.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124289 s, 844 MB/s 2024-04-26T04:43:10.025 INFO:teuthology.orchestra.run.smithi111.stderr:--> Zapping successful for: 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:10.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:10.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:09 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:43:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:10.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:09 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:10.808 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi111:vg_nvme/lv_1 2024-04-26T04:43:11.058 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:43:11.180 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:10 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi111 2024-04-26T04:43:11.180 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:10 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi111 to 5696M 2024-04-26T04:43:11.180 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:10 smithi111 ceph-mon[30255]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:43:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:10 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi111 2024-04-26T04:43:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:10 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi111 to 5696M 2024-04-26T04:43:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:10 smithi005 ceph-mon[26371]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:43:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:10 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi111 2024-04-26T04:43:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:10 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi111 to 5696M 2024-04-26T04:43:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:10 smithi005 ceph-mon[36961]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:43:12.122 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:11 smithi111 ceph-mon[30255]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:43:12.122 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:11 smithi111 ceph-mon[30255]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:11 smithi005 ceph-mon[26371]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:43:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:11 smithi005 ceph-mon[26371]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:11 smithi005 ceph-mon[36961]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:43:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:11 smithi005 ceph-mon[36961]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:13.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:12 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:43:13.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:12 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:43:13.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:12 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:13.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:12 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:43:13.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:12 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:43:13.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:12 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:13.249 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:12 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:43:13.249 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:12 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:43:13.249 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:12 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:14.173 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:13 smithi111 ceph-mon[30255]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:14.174 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:13 smithi111 ceph-mon[30255]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-26T04:43:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:13 smithi005 ceph-mon[26371]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:13 smithi005 ceph-mon[26371]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-26T04:43:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:13 smithi005 ceph-mon[36961]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi111:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:13 smithi005 ceph-mon[36961]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-26T04:43:16.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/2259365779' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]: dispatch 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]: dispatch 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]': finished 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/3259392380' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/2259365779' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]: dispatch 2024-04-26T04:43:16.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]: dispatch 2024-04-26T04:43:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]': finished 2024-04-26T04:43:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:43:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:16 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/3259392380' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/2259365779' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]: dispatch 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]: dispatch 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcc19600-f6e1-4c85-9f2a-8142ba69db4d"}]': finished 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:16 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/3259392380' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:43:18.457 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:18 smithi111 ceph-mon[30255]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:43:18.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:18 smithi005 ceph-mon[26371]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:43:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:18 smithi005 ceph-mon[36961]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:43:20.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:20 smithi111 ceph-mon[30255]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:43:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:20 smithi005 ceph-mon[26371]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:43:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:20 smithi005 ceph-mon[36961]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:43:22.114 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:22 smithi111 ceph-mon[30255]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:22.114 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:22 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:43:22.114 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:22 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:22 smithi005 ceph-mon[26371]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:22 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:43:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:22 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:22 smithi005 ceph-mon[36961]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:22 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:43:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:22 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:23.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:23 smithi005 ceph-mon[26371]: Deploying daemon osd.7 on smithi111 2024-04-26T04:43:23.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:23 smithi005 ceph-mon[36961]: Deploying daemon osd.7 on smithi111 2024-04-26T04:43:23.386 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:23 smithi111 ceph-mon[30255]: Deploying daemon osd.7 on smithi111 2024-04-26T04:43:24.263 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:24 smithi111 ceph-mon[30255]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:24 smithi005 ceph-mon[26371]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:24 smithi005 ceph-mon[36961]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:26.305 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:26 smithi111 ceph-mon[30255]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:26 smithi005 ceph-mon[26371]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:26 smithi005 ceph-mon[36961]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:27 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:27 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:27 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:27 smithi111 ceph-mon[30255]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[26371]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:27 smithi005 ceph-mon[36961]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:29.897 INFO:teuthology.orchestra.run.smithi111.stdout:Created osd(s) 7 on host 'smithi111' 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:43:30.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:43:30.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:30 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.506 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:30.507 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:30 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:30.533 DEBUG:teuthology.orchestra.run.smithi111:osd.7> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T04:43:30.535 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T04:43:30.536 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T04:43:30.810 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:43:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[26371]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:43:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[26371]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:43:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[26371]: from='osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[26371]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[36961]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:43:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[36961]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:43:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[36961]: from='osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:31 smithi005 ceph-mon[36961]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:31.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-mon[30255]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:43:31.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-mon[30255]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:43:31.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:31.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-mon[30255]: from='osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:31.488 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-mon[30255]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]: dispatch 2024-04-26T04:43:31.488 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T04:43:31.152+0000 7f1212911700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T04:43:31.488 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 04:43:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T04:43:31.160+0000 7f120c70a700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:43:32.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:32.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:43:32.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509] boot 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509] boot 2024-04-26T04:43:32.376 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:43:32.377 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:32 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:43:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi111", "root=default"]}]': finished 2024-04-26T04:43:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:43:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: osd.7 [v2:172.21.15.111:6824/3024259509,v1:172.21.15.111:6825/3024259509] boot 2024-04-26T04:43:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:43:32.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:32 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:32.792 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:43:33.206 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:33 smithi111 ceph-mon[30255]: purged_snaps scrub starts 2024-04-26T04:43:33.206 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:33 smithi111 ceph-mon[30255]: purged_snaps scrub ok 2024-04-26T04:43:33.206 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:33 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2620326946' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:43:33.206 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:33 smithi111 ceph-mon[30255]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:43:33.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[26371]: purged_snaps scrub starts 2024-04-26T04:43:33.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[26371]: purged_snaps scrub ok 2024-04-26T04:43:33.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2620326946' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:43:33.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[26371]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:43:33.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[36961]: purged_snaps scrub starts 2024-04-26T04:43:33.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[36961]: purged_snaps scrub ok 2024-04-26T04:43:33.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2620326946' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:43:33.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:33 smithi005 ceph-mon[36961]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:43:33.694 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1714106611,"num_in_osds":8,"osd_in_since":1714106595,"num_remapped_pgs":1} 2024-04-26T04:43:33.694 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:43:33.965 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:43:34.342 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:34.343 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.343 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.343 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.343 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.343 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.343 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.344 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.344 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:34.344 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:34.344 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.344 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[26371]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:43:34.345 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:34.345 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.345 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.345 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.345 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.346 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.346 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.346 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.346 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:34.346 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:34.346 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.347 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:34 smithi005 ceph-mon[36961]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:43:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:34.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:34.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:34.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:34.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:34 smithi111 ceph-mon[30255]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:43:35.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:35 smithi005 ceph-mon[26371]: Detected new or changed devices on smithi111 2024-04-26T04:43:35.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:35 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi111 to 4272M 2024-04-26T04:43:35.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:35 smithi005 ceph-mon[26371]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:43:35.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:35 smithi005 ceph-mon[36961]: Detected new or changed devices on smithi111 2024-04-26T04:43:35.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:35 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi111 to 4272M 2024-04-26T04:43:35.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:35 smithi005 ceph-mon[36961]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:43:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:35 smithi111 ceph-mon[30255]: Detected new or changed devices on smithi111 2024-04-26T04:43:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:35 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi111 to 4272M 2024-04-26T04:43:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:35 smithi111 ceph-mon[30255]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:43:35.929 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:43:35.929 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":59,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T04:43:35.907028+0000","last_up_change":"2024-04-26T04:43:31.898273+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6803","nonce":1943995172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6805","nonce":1943995172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6809","nonce":1943995172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6807","nonce":1943995172}]},"public_addr":"172.21.15.5:6803/1943995172","cluster_addr":"172.21.15.5:6805/1943995172","heartbeat_back_addr":"172.21.15.5:6809/1943995172","heartbeat_front_addr":"172.21.15.5:6807/1943995172","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6811","nonce":2192671947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6813","nonce":2192671947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6817","nonce":2192671947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6815","nonce":2192671947}]},"public_addr":"172.21.15.5:6811/2192671947","cluster_addr":"172.21.15.5:6813/2192671947","heartbeat_back_addr":"172.21.15.5:6817/2192671947","heartbeat_front_addr":"172.21.15.5:6815/2192671947","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6819","nonce":1951395348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6821","nonce":1951395348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6825","nonce":1951395348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6823","nonce":1951395348}]},"public_addr":"172.21.15.5:6819/1951395348","cluster_addr":"172.21.15.5:6821/1951395348","heartbeat_back_addr":"172.21.15.5:6825/1951395348","heartbeat_front_addr":"172.21.15.5:6823/1951395348","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6827","nonce":1448676980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6829","nonce":1448676980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6833","nonce":1448676980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6831","nonce":1448676980}]},"public_addr":"172.21.15.5:6827/1448676980","cluster_addr":"172.21.15.5:6829/1448676980","heartbeat_back_addr":"172.21.15.5:6833/1448676980","heartbeat_front_addr":"172.21.15.5:6831/1448676980","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:43:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:36 smithi005 ceph-mon[26371]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:36 smithi005 ceph-mon[26371]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:43:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:36 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3251895860' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:43:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:36 smithi005 ceph-mon[36961]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:36 smithi005 ceph-mon[36961]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:43:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:36 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3251895860' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:43:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:36 smithi111 ceph-mon[30255]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:36 smithi111 ceph-mon[30255]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:43:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:36 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3251895860' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:43:36.733 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T04:41:33.442542+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T04:43:36.733 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T04:43:37.007 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:43:38.429 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:38 smithi005 ceph-mon[36961]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:38.429 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:38 smithi005 ceph-mon[26371]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:38 smithi111 ceph-mon[30255]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:38.867 INFO:teuthology.orchestra.run.smithi005.stdout:pg_num: 1 2024-04-26T04:43:39.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:39 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1383974457' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:43:39.469 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:39 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1383974457' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:43:39.502 INFO:tasks.cephadm:Adding prometheus.a on smithi111 2024-04-26T04:43:39.503 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi111=a' 2024-04-26T04:43:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:39 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1383974457' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:43:39.762 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:43:40.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:40 smithi005 ceph-mon[26371]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:43:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:40 smithi005 ceph-mon[36961]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:43:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:40 smithi111 ceph-mon[30255]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:43:41.630 INFO:teuthology.orchestra.run.smithi111.stdout:Scheduled prometheus update... 2024-04-26T04:43:42.253 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:42 smithi111 ceph-mon[30255]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-26T04:43:42.253 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:42.253 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:42.253 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:42.253 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:42.253 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:42 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:42.277 DEBUG:teuthology.orchestra.run.smithi111:prometheus.a> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-26T04:43:42.319 INFO:tasks.cephadm:Adding node-exporter.a on smithi005 2024-04-26T04:43:42.319 INFO:tasks.cephadm:Adding node-exporter.b on smithi111 2024-04-26T04:43:42.320 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi005=a;smithi111=b' 2024-04-26T04:43:42.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[26371]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[36961]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:42.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:42 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:42.617 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:43:43.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:43 smithi005 ceph-mon[26371]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi111=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:43 smithi005 ceph-mon[26371]: Saving service prometheus spec with placement smithi111=a;count:1 2024-04-26T04:43:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:43 smithi005 ceph-mon[26371]: Deploying daemon prometheus.a on smithi111 2024-04-26T04:43:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:43 smithi005 ceph-mon[36961]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi111=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:43 smithi005 ceph-mon[36961]: Saving service prometheus spec with placement smithi111=a;count:1 2024-04-26T04:43:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:43 smithi005 ceph-mon[36961]: Deploying daemon prometheus.a on smithi111 2024-04-26T04:43:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:43 smithi111 ceph-mon[30255]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi111=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:43 smithi111 ceph-mon[30255]: Saving service prometheus spec with placement smithi111=a;count:1 2024-04-26T04:43:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:43 smithi111 ceph-mon[30255]: Deploying daemon prometheus.a on smithi111 2024-04-26T04:43:44.247 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:44 smithi111 ceph-mon[30255]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:43:44.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:44 smithi005 ceph-mon[26371]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:43:44.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:44 smithi005 ceph-mon[36961]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:43:44.994 INFO:teuthology.orchestra.run.smithi111.stdout:Scheduled node-exporter update... 2024-04-26T04:43:46.110 DEBUG:teuthology.orchestra.run.smithi005:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-26T04:43:46.113 DEBUG:teuthology.orchestra.run.smithi111:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-26T04:43:46.115 INFO:tasks.cephadm:Adding alertmanager.a on smithi005 2024-04-26T04:43:46.115 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi005=a' 2024-04-26T04:43:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:45 smithi111 ceph-mon[30255]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi005=a;smithi111=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:45 smithi111 ceph-mon[30255]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:45 smithi111 ceph-mon[30255]: Saving service node-exporter spec with placement smithi005=a;smithi111=b;count:2 2024-04-26T04:43:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:45 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:46.399 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:43:46.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[26371]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi005=a;smithi111=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[26371]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[26371]: Saving service node-exporter spec with placement smithi005=a;smithi111=b;count:2 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[36961]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi005=a;smithi111=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[36961]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[36961]: Saving service node-exporter spec with placement smithi005=a;smithi111=b;count:2 2024-04-26T04:43:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:45 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:47 smithi111 ceph-mon[30255]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T04:43:48.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:47 smithi005 ceph-mon[26371]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T04:43:48.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:47 smithi005 ceph-mon[36961]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T04:43:49.012 INFO:teuthology.orchestra.run.smithi111.stdout:Scheduled alertmanager update... 2024-04-26T04:43:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:50 smithi111 ceph-mon[30255]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:43:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:50 smithi111 ceph-mon[30255]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi005=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:50 smithi111 ceph-mon[30255]: Saving service alertmanager spec with placement smithi005=a;count:1 2024-04-26T04:43:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:50 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:50.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[26371]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[26371]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi005=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[26371]: Saving service alertmanager spec with placement smithi005=a;count:1 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[36961]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[36961]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi005=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[36961]: Saving service alertmanager spec with placement smithi005=a;count:1 2024-04-26T04:43:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:50 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:50.539 DEBUG:teuthology.orchestra.run.smithi005:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-26T04:43:50.542 INFO:tasks.cephadm:Adding grafana.a on smithi111 2024-04-26T04:43:50.543 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi111=a' 2024-04-26T04:43:50.825 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:43:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:52 smithi111 ceph-mon[30255]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:52.335 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:52 smithi005 ceph-mon[26371]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:52.335 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:52 smithi005 ceph-mon[36961]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:54 smithi111 ceph-mon[30255]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:54 smithi005 ceph-mon[36961]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:54.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:54 smithi005 ceph-mon[26371]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:56.300 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:56 smithi111 ceph-mon[30255]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:56.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:56 smithi005 ceph-mon[26371]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:56 smithi005 ceph-mon[36961]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:56.919 INFO:teuthology.orchestra.run.smithi111.stdout:Scheduled grafana update... 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi111=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: Saving service grafana spec with placement smithi111=a;count:1 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.002 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:57 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:43:58.025 DEBUG:teuthology.orchestra.run.smithi111:grafana.a> sudo journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service 2024-04-26T04:43:58.028 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T04:43:58.029 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:43:58.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi111=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:58.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: Saving service grafana spec with placement smithi111=a;count:1 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi111=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: Saving service grafana spec with placement smithi111=a;count:1 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' 2024-04-26T04:43:58.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:57 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:43:58.301 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:43:58.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setuser ceph since I am not root 2024-04-26T04:43:58.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setgroup ceph since I am not root 2024-04-26T04:43:58.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:58 smithi005 ceph-mgr[26603]: -- 172.21.15.5:0/4045149714 <== mon.2 v2:172.21.15.5:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56166eb1c9c0 con 0x56166eaf2c00 2024-04-26T04:43:58.924 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ignoring --setuser ceph since I am not root 2024-04-26T04:43:58.924 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ignoring --setgroup ceph since I am not root 2024-04-26T04:43:58.924 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:58 smithi111 ceph-mgr[32164]: -- 172.21.15.111:0/2022071612 <== mon.1 v2:172.21.15.111:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x5591a0116c60 con 0x5591a00d8800 2024-04-26T04:43:58.925 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:58 smithi111 ceph-mgr[32164]: -- 172.21.15.111:0/2022071612 <== mon.1 v2:172.21.15.111:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5591a01029c0 con 0x5591a00d8800 2024-04-26T04:43:58.925 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:43:58.765+0000 7f2a4b3b21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:43:58.925 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:43:58.923+0000 7f2a4b3b21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:59.042 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:43:58.770+0000 7ff806e0e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:43:59.043 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:43:58.936+0000 7ff806e0e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:59.043 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:43:59.041+0000 7ff806e0e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:43:59.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:43:59.021+0000 7f2a4b3b21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:43:59.602 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:43:59.319+0000 7f2a4b3b21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:43:59.602 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:43:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:43:59.400+0000 7f2a4b3b21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:43:59.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-mon[36961]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:43:59.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-mon[36961]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:43:59.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-mon[26371]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:43:59.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-mon[26371]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:43:59.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:43:59.359+0000 7ff806e0e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:43:59.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:43:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:43:59.445+0000 7ff806e0e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:44:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:59 smithi111 ceph-mon[30255]: from='mgr.14152 172.21.15.5:0/505203370' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:44:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:43:59 smithi111 ceph-mon[30255]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:44:00.410 INFO:teuthology.orchestra.run.smithi005.stdout:[client.0] 2024-04-26T04:44:00.410 INFO:teuthology.orchestra.run.smithi005.stdout: key = AQAQMStmk/EvGBAAOTLPy0wYiPG5XDENIGHE/Q== 2024-04-26T04:44:00.686 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:00 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1753836909' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:00.686 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:00 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1753836909' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:44:00.687 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:00 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1753836909' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:00.687 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:00 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1753836909' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:44:00.900 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:00 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1753836909' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:00.900 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:00 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1753836909' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:44:00.900 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:00.555+0000 7f2a4b3b21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:44:00.900 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:00.727+0000 7f2a4b3b21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:00.900 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:00.898+0000 7f2a4b3b21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:44:00.981 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:00.685+0000 7ff806e0e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:44:00.981 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:00.858+0000 7ff806e0e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:01.165 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:44:01.165 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T04:44:01.165 DEBUG:teuthology.orchestra.run.smithi005:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T04:44:01.202 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:44:01.311 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:01.032+0000 7ff806e0e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:44:01.469 INFO:teuthology.orchestra.run.smithi111.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:44:01.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:01.167+0000 7f2a4b3b21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:44:01.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:01.262+0000 7f2a4b3b21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:44:01.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:01.361+0000 7f2a4b3b21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:44:01.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:01.310+0000 7ff806e0e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:44:01.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:01.403+0000 7ff806e0e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:44:01.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:01.501+0000 7ff806e0e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:44:01.863 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:01.656+0000 7f2a4b3b21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:02.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:01.788+0000 7ff806e0e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:02.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:02.119+0000 7f2a4b3b21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:44:02.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:02 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:02.271+0000 7ff806e0e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:44:02.818 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:02.653+0000 7f2a4b3b21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:44:02.818 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:02.747+0000 7f2a4b3b21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:44:03.102 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:02 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:02.817+0000 7ff806e0e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:44:03.103 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:02 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:02.915+0000 7ff806e0e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:44:03.103 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:03.101+0000 7ff806e0e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:44:03.244 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:02.932+0000 7f2a4b3b21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:44:03.244 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:03.242+0000 7f2a4b3b21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:44:03.305 INFO:teuthology.orchestra.run.smithi111.stdout:[client.1] 2024-04-26T04:44:03.305 INFO:teuthology.orchestra.run.smithi111.stdout: key = AQATMStmLI3zERAAVQg17C+nBnU/FSZ7eymwsA== 2024-04-26T04:44:03.510 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-mon[30255]: from='client.? 172.21.15.111:0/860881036' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:03.510 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:03.510 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:44:03.510 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:03.508+0000 7f2a4b3b21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:44:03.677 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-mon[26371]: from='client.? 172.21.15.111:0/860881036' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:03.677 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:03.677 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:44:03.677 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-mon[36961]: from='client.? 172.21.15.111:0/860881036' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:03.677 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:44:03.677 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:44:03.678 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:03.409+0000 7ff806e0e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:44:03.678 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:03.675+0000 7ff806e0e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:44:03.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:03.752+0000 7f2a4b3b21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:44:03.929 DEBUG:teuthology.orchestra.run.smithi111:> set -ex 2024-04-26T04:44:03.930 DEBUG:teuthology.orchestra.run.smithi111:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-26T04:44:03.930 DEBUG:teuthology.orchestra.run.smithi111:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-26T04:44:03.966 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-26T04:44:03.967 DEBUG:teuthology.orchestra.run.smithi005:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-26T04:44:03.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:03.925+0000 7ff806e0e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:44:04.190 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:03.887+0000 7f2a4b3b21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:44:04.191 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:03.987+0000 7f2a4b3b21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:44:04.191 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:04.101+0000 7f2a4b3b21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:04.191 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:04.189+0000 7f2a4b3b21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:44:04.270 INFO:teuthology.misc.health.smithi005.stdout: 2024-04-26T04:44:04.271 INFO:teuthology.misc.health.smithi005.stdout:{"epoch":59,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T04:43:35.907028+0000","last_up_change":"2024-04-26T04:43:31.898273+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6803","nonce":1943995172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6805","nonce":1943995172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6809","nonce":1943995172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6807","nonce":1943995172}]},"public_addr":"172.21.15.5:6803/1943995172","cluster_addr":"172.21.15.5:6805/1943995172","heartbeat_back_addr":"172.21.15.5:6809/1943995172","heartbeat_front_addr":"172.21.15.5:6807/1943995172","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6811","nonce":2192671947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6813","nonce":2192671947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6817","nonce":2192671947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6815","nonce":2192671947}]},"public_addr":"172.21.15.5:6811/2192671947","cluster_addr":"172.21.15.5:6813/2192671947","heartbeat_back_addr":"172.21.15.5:6817/2192671947","heartbeat_front_addr":"172.21.15.5:6815/2192671947","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6819","nonce":1951395348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6821","nonce":1951395348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6825","nonce":1951395348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6823","nonce":1951395348}]},"public_addr":"172.21.15.5:6819/1951395348","cluster_addr":"172.21.15.5:6821/1951395348","heartbeat_back_addr":"172.21.15.5:6825/1951395348","heartbeat_front_addr":"172.21.15.5:6823/1951395348","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6827","nonce":1448676980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6829","nonce":1448676980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6833","nonce":1448676980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6831","nonce":1448676980}]},"public_addr":"172.21.15.5:6827/1448676980","cluster_addr":"172.21.15.5:6829/1448676980","heartbeat_back_addr":"172.21.15.5:6833/1448676980","heartbeat_front_addr":"172.21.15.5:6831/1448676980","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:44:04.282 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-26T04:44:04.282 INFO:tasks.cephadm:Creating RBD pool 2024-04-26T04:44:04.282 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-26T04:44:04.343 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:04.066+0000 7ff806e0e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:44:04.343 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:04.169+0000 7ff806e0e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:44:04.343 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:04.292+0000 7ff806e0e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:04.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:04 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2849952238' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:04.587 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:04.710 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:04 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2849952238' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:04.710 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:04 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2849952238' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:04.710 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:04.388+0000 7ff806e0e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:44:05.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:04.983+0000 7f2a4b3b21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:44:05.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:05.172+0000 7f2a4b3b21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:44:05.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: [26/Apr/2024:04:44:05] ENGINE Bus STARTING 2024-04-26T04:44:05.356 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:05.250+0000 7ff806e0e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-mon[30255]: Standby manager daemon x restarted 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-mon[30255]: Standby manager daemon x started 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: CherryPy Checker: 2024-04-26T04:44:05.564 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: The Application mounted at '' has an empty config. 2024-04-26T04:44:05.565 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: 2024-04-26T04:44:05.565 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: [26/Apr/2024:04:44:05] ENGINE Serving on http://:::9283 2024-04-26T04:44:05.565 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: [26/Apr/2024:04:44:05] ENGINE Bus STARTED 2024-04-26T04:44:05.625 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[26371]: Standby manager daemon x restarted 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[26371]: Standby manager daemon x started 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[36961]: Standby manager daemon x restarted 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[36961]: Standby manager daemon x started 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:05.626 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:05.627 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:05.627 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2953819894' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:05.627 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:05.458+0000 7ff806e0e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:44:05.627 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:05] ENGINE Bus STARTING 2024-04-26T04:44:05.909 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: CherryPy Checker: 2024-04-26T04:44:05.909 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: The Application mounted at '' has an empty config. 2024-04-26T04:44:05.909 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:44:05.909 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:05] ENGINE Serving on http://:::9283 2024-04-26T04:44:05.909 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:05] ENGINE Bus STARTED 2024-04-26T04:44:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: Active manager daemon y restarted 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: Activating manager daemon y 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: mgrmap e18: y(active, starting, since 0.00626887s), standbys: x 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:44:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: Manager daemon y is now available 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:06.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: Active manager daemon y restarted 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: Activating manager daemon y 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: mgrmap e18: y(active, starting, since 0.00626887s), standbys: x 2024-04-26T04:44:06.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:44:06.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: Manager daemon y is now available 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:06.485 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:06.616 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-26T04:44:06.616 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: Active manager daemon y restarted 2024-04-26T04:44:06.616 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: Activating manager daemon y 2024-04-26T04:44:06.616 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:44:06.617 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: mgrmap e18: y(active, starting, since 0.00626887s), standbys: x 2024-04-26T04:44:06.617 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:44:06.617 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:44:06.617 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:44:06.617 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:44:06.617 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:44:06.618 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:44:06.618 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:44:06.618 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:44:06.618 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:44:06.618 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:44:06.618 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:44:06.619 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:44:06.619 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:44:06.619 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:44:06.619 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:44:06.619 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:44:06.619 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: Manager daemon y is now available 2024-04-26T04:44:06.620 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:06.620 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:06.620 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:06.620 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:06.620 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:06.620 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:06.621 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:07.481 INFO:teuthology.orchestra.run.smithi005.stderr:pool 'rbd' created 2024-04-26T04:44:07.657 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: [26/Apr/2024:04:44:05] ENGINE Bus STARTING 2024-04-26T04:44:07.658 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: [26/Apr/2024:04:44:05] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:44:07.658 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: [26/Apr/2024:04:44:05] ENGINE Bus STARTED 2024-04-26T04:44:07.658 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: mgrmap e19: y(active, since 1.01334s), standbys: x 2024-04-26T04:44:07.658 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:07.658 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3487533378' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:44:07.659 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:07.659 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:07.659 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: [26/Apr/2024:04:44:05] ENGINE Bus STARTING 2024-04-26T04:44:07.659 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: [26/Apr/2024:04:44:05] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:44:07.659 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: [26/Apr/2024:04:44:05] ENGINE Bus STARTED 2024-04-26T04:44:07.660 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: mgrmap e19: y(active, since 1.01334s), standbys: x 2024-04-26T04:44:07.660 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:07.660 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3487533378' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:44:07.660 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:07.660 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: [26/Apr/2024:04:44:05] ENGINE Bus STARTING 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: [26/Apr/2024:04:44:05] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: [26/Apr/2024:04:44:05] ENGINE Bus STARTED 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: mgrmap e19: y(active, since 1.01334s), standbys: x 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3487533378' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:44:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:07.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.215 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3487533378' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-26T04:44:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:08.484 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:08.541 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3487533378' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:08.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3487533378' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:44:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:44:08.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:08.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi111 to 4016M 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi111 to 4016M 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.753 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.753 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:09 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi111 to 4016M 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:09 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: Deploying daemon alertmanager.a on smithi005 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:44:10.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[26371]: mgrmap e21: y(active, since 4s), standbys: x 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: Deploying daemon alertmanager.a on smithi005 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:44:10.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:10 smithi005 ceph-mon[36961]: mgrmap e21: y(active, since 4s), standbys: x 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: Deploying daemon alertmanager.a on smithi005 2024-04-26T04:44:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:10.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:44:10.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:10 smithi111 ceph-mon[30255]: mgrmap e21: y(active, since 4s), standbys: x 2024-04-26T04:44:11.496 INFO:teuthology.orchestra.run.smithi005.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-26T04:44:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:11 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/756982136' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:44:11.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:44:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:11.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:11 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/756982136' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:44:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:11 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/756982136' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:44:12.677 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:44:12.677 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:44:12.677 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:44:12.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:12 smithi111 ceph-mon[30255]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:12.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:12 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/756982136' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:44:12.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:12 smithi111 ceph-mon[30255]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:44:12.964 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:12 smithi005 ceph-mon[26371]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:12 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/756982136' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:44:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:12 smithi005 ceph-mon[26371]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:44:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:12 smithi005 ceph-mon[36961]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:12 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/756982136' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:44:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:12 smithi005 ceph-mon[36961]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:44:13.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:13 smithi005 ceph-mon[36961]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:44:13.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:13 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:44:13.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:13 smithi005 ceph-mon[26371]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:44:13.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:13 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:44:13.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:13 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:44:13.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:13 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:44:14.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:14 smithi005 ceph-mon[26371]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:44:14.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:14 smithi005 ceph-mon[36961]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:44:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:14 smithi111 ceph-mon[30255]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:44:16.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:16 smithi005 ceph-mon[26371]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:44:16.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:16 smithi005 ceph-mon[36961]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:44:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:16 smithi111 ceph-mon[30255]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:44:17.465 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:44:17.649 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:17 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3405420390' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:44:17.651 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:17 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3405420390' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:17 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3405420390' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.693 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:18 smithi005 ceph-mon[36961]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:44:18.693 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:18 smithi005 ceph-mon[26371]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:44:18.724 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":21,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6800","nonce":1913329511},{"type":"v1","addr":"172.21.15.5:6801","nonce":1913329511}]},"active_addr":"172.21.15.5:6801/1913329511","active_change":"2024-04-26T04:44:05.461138+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24373,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.5:8443/","prometheus":"http://172.21.15.5:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":4008904229}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":2166298037}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":188422345}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":4211393178}]}]} 2024-04-26T04:44:18.728 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:44:18.728 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:44:18.728 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:44:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:18 smithi111 ceph-mon[30255]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:44:19.429 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:20.663 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:20 smithi005 ceph-mon[26371]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:20.663 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:20 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:20.663 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:20 smithi005 ceph-mon[36961]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:20.663 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:20 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:20 smithi111 ceph-mon[30255]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:20 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:21.382 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:44:21.382 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":64,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T04:44:11.484478+0000","last_up_change":"2024-04-26T04:43:31.898273+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:44:07.074856+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6803","nonce":1943995172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6805","nonce":1943995172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6809","nonce":1943995172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6807","nonce":1943995172}]},"public_addr":"172.21.15.5:6803/1943995172","cluster_addr":"172.21.15.5:6805/1943995172","heartbeat_back_addr":"172.21.15.5:6809/1943995172","heartbeat_front_addr":"172.21.15.5:6807/1943995172","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6811","nonce":2192671947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6813","nonce":2192671947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6817","nonce":2192671947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6815","nonce":2192671947}]},"public_addr":"172.21.15.5:6811/2192671947","cluster_addr":"172.21.15.5:6813/2192671947","heartbeat_back_addr":"172.21.15.5:6817/2192671947","heartbeat_front_addr":"172.21.15.5:6815/2192671947","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6819","nonce":1951395348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6821","nonce":1951395348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6825","nonce":1951395348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6823","nonce":1951395348}]},"public_addr":"172.21.15.5:6819/1951395348","cluster_addr":"172.21.15.5:6821/1951395348","heartbeat_back_addr":"172.21.15.5:6825/1951395348","heartbeat_front_addr":"172.21.15.5:6823/1951395348","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6827","nonce":1448676980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6829","nonce":1448676980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6833","nonce":1448676980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6831","nonce":1448676980}]},"public_addr":"172.21.15.5:6827/1448676980","cluster_addr":"172.21.15.5:6829/1448676980","heartbeat_back_addr":"172.21.15.5:6833/1448676980","heartbeat_front_addr":"172.21.15.5:6831/1448676980","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/1382772406":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1663053230":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/3920149860":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3928321375":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:44:21.824 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:21 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1593661141' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:21.824 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:21 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1593661141' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:21.824 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:44:21 smithi005 systemd[1]: Started Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:21 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1593661141' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:22.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:44:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:22.158 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:21] ENGINE Bus STOPPING 2024-04-26T04:44:22.158 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:22.158 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:21] ENGINE Bus STOPPED 2024-04-26T04:44:22.158 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:21] ENGINE Bus STARTING 2024-04-26T04:44:22.159 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:22] ENGINE Serving on http://:::9283 2024-04-26T04:44:22.159 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:44:22] ENGINE Bus STARTED 2024-04-26T04:44:22.547 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:44:22.547 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:44:22.813 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:44:22.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:44:22.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:44:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:23.962 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:44:23.963 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:23 smithi005 ceph-mon[26371]: Deploying daemon grafana.a on smithi111 2024-04-26T04:44:23.963 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:44:23.963 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:23 smithi005 ceph-mon[36961]: Deploying daemon grafana.a on smithi111 2024-04-26T04:44:23.963 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:44:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[66571]: ts=2024-04-26T04:44:23.679Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000486329s 2024-04-26T04:44:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:44:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:23 smithi111 ceph-mon[30255]: Deploying daemon grafana.a on smithi111 2024-04-26T04:44:24.729 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:44:24.730 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":64,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T04:44:11.484478+0000","last_up_change":"2024-04-26T04:43:31.898273+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:44:07.074856+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6803","nonce":1943995172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6805","nonce":1943995172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6809","nonce":1943995172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6807","nonce":1943995172}]},"public_addr":"172.21.15.5:6803/1943995172","cluster_addr":"172.21.15.5:6805/1943995172","heartbeat_back_addr":"172.21.15.5:6809/1943995172","heartbeat_front_addr":"172.21.15.5:6807/1943995172","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6811","nonce":2192671947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6813","nonce":2192671947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6817","nonce":2192671947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6815","nonce":2192671947}]},"public_addr":"172.21.15.5:6811/2192671947","cluster_addr":"172.21.15.5:6813/2192671947","heartbeat_back_addr":"172.21.15.5:6817/2192671947","heartbeat_front_addr":"172.21.15.5:6815/2192671947","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6819","nonce":1951395348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6821","nonce":1951395348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6825","nonce":1951395348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6823","nonce":1951395348}]},"public_addr":"172.21.15.5:6819/1951395348","cluster_addr":"172.21.15.5:6821/1951395348","heartbeat_back_addr":"172.21.15.5:6825/1951395348","heartbeat_front_addr":"172.21.15.5:6823/1951395348","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6827","nonce":1448676980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6829","nonce":1448676980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6833","nonce":1448676980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6831","nonce":1448676980}]},"public_addr":"172.21.15.5:6827/1448676980","cluster_addr":"172.21.15.5:6829/1448676980","heartbeat_back_addr":"172.21.15.5:6833/1448676980","heartbeat_front_addr":"172.21.15.5:6831/1448676980","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/1382772406":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1663053230":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/3920149860":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3928321375":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:44:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:24 smithi005 ceph-mon[26371]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:24 smithi005 ceph-mon[36961]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:24 smithi111 ceph-mon[30255]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:25.438 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:44:25.439 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:44:25.439 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:44:25.439 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:44:25.440 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:44:25.440 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:44:25.441 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:44:25.441 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:44:25.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:25 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/610275127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:25.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:25 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:25.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:25 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/610275127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:25.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:25 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:25.990 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:25.991 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:25.995 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:25.995 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:26.000 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:26.002 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:26.003 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:25 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/610275127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:44:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:25 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:26.019 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:26.673 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:26 smithi005 ceph-mon[26371]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:26.674 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:26 smithi005 ceph-mon[36961]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:26 smithi111 ceph-mon[30255]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:28 smithi005 ceph-mon[36961]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:28.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:28 smithi005 ceph-mon[26371]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:28 smithi111 ceph-mon[30255]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:30.716 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:30 smithi005 ceph-mon[26371]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:30.717 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:30 smithi005 ceph-mon[36961]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:30 smithi111 ceph-mon[30255]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:31.859 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:44:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[66571]: ts=2024-04-26T04:44:31.683Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004001835s 2024-04-26T04:44:32.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:44:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:32.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:32 smithi111 ceph-mon[30255]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:32.802 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:32 smithi005 ceph-mon[26371]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:32.803 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:32 smithi005 ceph-mon[36961]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:34.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:34 smithi005 ceph-mon[26371]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:34.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:34 smithi005 ceph-mon[36961]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:34 smithi111 ceph-mon[30255]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:35.460 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:35.712 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460024596Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460207502Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460274822Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460315747Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460326219Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460337256Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460348279Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460360927Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460372332Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460383055Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460393727Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460404078Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460415015Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-26T04:44:35.713 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=settings t=2024-04-26T04:44:35.460426335Z level=info msg="App mode production" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=sqlstore t=2024-04-26T04:44:35.460480624Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=sqlstore t=2024-04-26T04:44:35.460503916Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.461432028Z level=info msg="Starting DB migrations" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.461518054Z level=info msg="Executing migration" id="create migration_log table" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.462068273Z level=info msg="Executing migration" id="create user table" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.462537564Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.463002385Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.463462251Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.464047839Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.464463423Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.465170579Z level=info msg="Executing migration" id="create user table v2" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.465578943Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.46595738Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-26T04:44:35.714 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.466419786Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.466762641Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.467138091Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.467557519Z level=info msg="Executing migration" id="Update user table charset" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.46783472Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.468268045Z level=info msg="Executing migration" id="Add missing user data" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.468642507Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.469050571Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.472099067Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.472531844Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-26T04:44:35.715 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.473650171Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-26T04:44:35.717 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.474082364Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.474530527Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.474927317Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.475356755Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.475754067Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.476029152Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.476421378Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.476808267Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.477203001Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-26T04:44:35.718 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.477600555Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.47823041Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.47867371Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.479050904Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.479446114Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.479837185Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.480250376Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.480643417Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.480968804Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.481306599Z level=info msg="Executing migration" id="create star table" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.48167764Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.482083479Z level=info msg="Executing migration" id="create org table v1" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.482477692Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.484788809Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-26T04:44:35.719 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.485188686Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.485661029Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.4860725Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.48649658Z level=info msg="Executing migration" id="Update org table charset" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.486768366Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.48704623Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.487364424Z level=info msg="Executing migration" id="create dashboard table" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.487758297Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.488158408Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.488560386Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.488971767Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.491711704Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.492141818Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.494938897Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-26T04:44:35.720 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.497558485Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.500215169Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.500711224Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.501039808Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.503514952Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.503844427Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.506238937Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.508970107Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.511622835Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.514346769Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.51687362Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.519734072Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.522214305Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.525028118Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-26T04:44:35.721 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.525412864Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.528376826Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.528969352Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.529890345Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.530612921Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.531464907Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.532106359Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.532548103Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.53329918Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.533791587Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.534215448Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.535139333Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.53609697Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.536744217Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-26T04:44:35.722 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.537564055Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.538421553Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.539343443Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.540067653Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.540850369Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.541750312Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.542456249Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.543343226Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.544194502Z level=info msg="Executing migration" id="create data_source table" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.545033953Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.545813711Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.546266033Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.547103496Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.548081015Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-26T04:44:35.723 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.549235523Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.550104828Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.55093485Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.551849042Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.552582779Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.553446689Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.554403172Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.555124147Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.556005966Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.556823239Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.557816548Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.560412245Z level=info msg="Executing migration" id="Add uid column" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.562969473Z level=info msg="Executing migration" id="Update uid value" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.565705444Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-26T04:44:35.724 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.5684148Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.571298453Z level=info msg="Executing migration" id="create api_key table" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.573970544Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.576534091Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.57940736Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.582080597Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.584801474Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.587295591Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.587744262Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.590339352Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.59310646Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.595592434Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.596058336Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.598782141Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.601207843Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-26T04:44:35.725 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.6018585Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.602180644Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.603071014Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.604050845Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.604822415Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.605401443Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.608180886Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.610742226Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.613724479Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.614280823Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.617133485Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.619774794Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.622388379Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.622729727Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-26T04:44:35.726 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.62556962Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.626205698Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.628361383Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.631113291Z level=info msg="Executing migration" id="create quota table v1" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.631531267Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.634266054Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.637043377Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.63993639Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.642622024Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.645179964Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.647712926Z level=info msg="Executing migration" id="create session table" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.650221874Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.650556735Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.653370592Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-26T04:44:35.727 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.653929334Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.656102279Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.65850304Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.658811418Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.661480567Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.664229648Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.666757434Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.66725861Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-26T04:44:35.728 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.669900489Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.6727193Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.675332187Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.676008703Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.678731859Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.679626822Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.680296451Z level=info msg="Executing migration" id="create alert table v1" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.680810516Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.681570645Z level=info msg="Executing migration" id="add index alert state" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.68251155Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.683156689Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.683525406Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.684338485Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.685222874Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-26T04:44:35.729 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.686859566Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.687240871Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.688060698Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.688937365Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.689623525Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.690000139Z level=info msg="Executing migration" id="Add column is_default" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.690670452Z level=info msg="Executing migration" id="Add column frequency" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.691371517Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.692152205Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.693044141Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.693508717Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.694324286Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.694686478Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.695176725Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-26T04:44:35.730 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.69603313Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.696462227Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.697274813Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.697756711Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.698466695Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.699303712Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.699684159Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.700607838Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.701369534Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.702098912Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.702447871Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.703253697Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.703760472Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.706029775Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.706488267Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-26T04:44:35.731 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.708645604Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-26T04:44:35.732 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.709126062Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-26T04:44:35.732 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.711818034Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-26T04:44:35.732 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.714570131Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-26T04:44:35.732 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.71726631Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.717645391Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.720321701Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.722998187Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.725767954Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.726183362Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.729045038Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.731744479Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.734350924Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-26T04:44:35.963 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.734892962Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.737749001Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.740239082Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.743385952Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.743765272Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.746381826Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.749078794Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.751752671Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.752228613Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.754850438Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.757503403Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.760169598Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.760655481Z level=info msg="Executing migration" id="Move region to single row" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.763174561Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.763574219Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-26T04:44:35.964 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.765720971Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.766171377Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.768485446Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.768898157Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.769668474Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.770503292Z level=info msg="Executing migration" id="create test_data table" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.771324199Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.771723027Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.772624338Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.773300398Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.77407853Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.774403436Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.775304463Z level=info msg="Executing migration" id="create team table" 2024-04-26T04:44:35.965 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.775855901Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.776298604Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.777099174Z level=info msg="Executing migration" id="create team member table" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.77748509Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.778254205Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.779145369Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.779814058Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.780538061Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.781316151Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-26T04:44:35.967 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.782169688Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.782970964Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.783461976Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.784308073Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.78521831Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.786164502Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.786694364Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.787558823Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.788351815Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.789283173Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.789778622Z level=info msg="Executing migration" id="create tag table" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.790604534Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.791547334Z level=info msg="Executing migration" id="create login attempt table" 2024-04-26T04:44:35.968 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.792195844Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.793164162Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.793836264Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.795663479Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.796436768Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.797432285Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.798104058Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.799089346Z level=info msg="Executing migration" id="create user auth table" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.799802095Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.800667987Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-26T04:44:35.969 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.801407551Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.802215751Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.803011208Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.803807496Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.804679447Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.805655762Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.806587983Z level=info msg="Executing migration" id="create server_lock table" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.807434816Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.808379783Z level=info msg="Executing migration" id="create user auth token table" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.809118667Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.809959908Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.810813658Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.811804826Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.812761511Z level=info msg="Executing migration" id="create cache_data table" 2024-04-26T04:44:35.970 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.813538241Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.814420271Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.815361536Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.816102691Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.81639066Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.81665609Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.817086634Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.817891425Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.818313723Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.818598911Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.818973215Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.819398351Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.819788344Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.820201115Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-26T04:44:35.971 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.820972952Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.821374904Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.821641287Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.822013044Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.82242394Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.822803833Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.823060574Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.823445535Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.823819379Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.82422963Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.824636047Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.825419838Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.825818107Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.82622009Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.829603509Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-26T04:44:35.972 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.832851295Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.833284161Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.833693762Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.834534632Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.834937797Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.835385806Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.835792659Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.836207012Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.836513296Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.83730804Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.838145357Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.838942085Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.839348613Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.839729781Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-26T04:44:35.973 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.840556488Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.841357735Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.841786635Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.842611016Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.843445693Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.843748928Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.84420397Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.845057006Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.845505716Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.845774415Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.846662918Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.84855922Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.85134489Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.853902602Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-26T04:44:35.974 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.856772591Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.859298893Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.862087718Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.864776202Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.867264676Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.870100038Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.872565761Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.875145771Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.877911985Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.880526562Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.883385123Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.885943332Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.888638648Z level=info msg="Executing migration" id="create alert_image table" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.89113074Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-26T04:44:35.975 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.893796399Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.896489866Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.898913365Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.899327069Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.899402211Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.902074468Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.904554169Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.907055581Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.90716849Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.907272348Z level=info msg="alerts found to migrate" alerts=0 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.909672941Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.91233377Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.91487189Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.917521818Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-26T04:44:35.976 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.919976085Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.922746934Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.925196078Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.927754558Z level=info msg="Executing migration" id="create data_keys table" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.930490124Z level=info msg="Executing migration" id="create secrets table" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.932918436Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.936997055Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.937892094Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.938691168Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.942870428Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.946848891Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.947668794Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.948447569Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.949240578Z level=info msg="Executing migration" id="create permission table" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.950183898Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-26T04:44:35.977 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.950856194Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.951859355Z level=info msg="Executing migration" id="create role table" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.952516245Z level=info msg="Executing migration" id="add column display_name" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.953443553Z level=info msg="Executing migration" id="add column group_name" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.954362614Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.955190276Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.956099721Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.956765345Z level=info msg="Executing migration" id="create team role table" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.957556225Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.958418585Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.959245552Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.96020812Z level=info msg="Executing migration" id="create user role table" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.960970265Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.961908797Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-26T04:44:35.978 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.962395179Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-26T04:44:35.979 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.963251484Z level=info msg="Executing migration" id="create builtin role table" 2024-04-26T04:44:35.979 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.96408403Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-26T04:44:35.979 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.964893331Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-26T04:44:35.979 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.965678797Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-26T04:44:36.268 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.967029557Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.967812575Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.968620003Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.96949577Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.972176758Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.974914654Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.977728728Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.980384923Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.983007563Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.985762354Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.988690743Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.991191164Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-26T04:44:36.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.993967795Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-26T04:44:36.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.996894693Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-26T04:44:36.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:35.999443715Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-26T04:44:36.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.002283249Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-26T04:44:36.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.004978277Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-26T04:44:36.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.00777758Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-26T04:44:36.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.010424952Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.01329215Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.016027912Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.018795564Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.021566588Z level=info msg="Executing migration" id="add correlation config column" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.024165677Z level=info msg="Executing migration" id="create entity_events table" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.026802295Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.029619669Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.029727787Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.032134252Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.032230752Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:44:36.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.03476657Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.037269281Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.039977771Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.042586475Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.045311634Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.047948874Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.050588748Z level=info msg="Executing migration" id="Drop public config table" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.053303319Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.055894017Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.058529565Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.061287053Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.063827104Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.06674479Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.067784741Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.068814743Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-26T04:44:36.272 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.069738037Z level=info msg="Executing migration" id="add share column" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.070826864Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.071638581Z level=info msg="Executing migration" id="create file table" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.072630258Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.07328697Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.07406088Z level=info msg="Executing migration" id="create file_meta table" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.07501021Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.075663488Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.076586045Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.077055448Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.077553317Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.078326804Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.07936243Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.079676393Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-26T04:44:36.273 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.080485452Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.081351427Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.082129135Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.0830786Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.084202093Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.085200333Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.086065302Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.09580893Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.096641675Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.097505932Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.101082127Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.102025545Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.102697699Z level=info msg="Executing migration" id="create folder table" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.103512955Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-26T04:44:36.274 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.103969598Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.104422006Z level=info msg="Executing migration" id="Update folder title length" 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.105293006Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=migrator t=2024-04-26T04:44:36.105740595Z level=info msg="migrations completed" performed=485 skipped=0 duration=644.243294ms 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=sqlstore t=2024-04-26T04:44:36.106135314Z level=info msg="Created default organization" 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=plugin.loader t=2024-04-26T04:44:36.162237115Z level=info msg="Plugin registered" pluginID=input 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=plugin.loader t=2024-04-26T04:44:36.189221567Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=plugin.loader t=2024-04-26T04:44:36.189238841Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=secrets t=2024-04-26T04:44:36.18928957Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=query_data t=2024-04-26T04:44:36.190930538Z level=info msg="Query Service initialization" 2024-04-26T04:44:36.275 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=live.push_http t=2024-04-26T04:44:36.193300068Z level=info msg="Live Push Gateway initialization" 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: Deploying daemon node-exporter.a on smithi005 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.531 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.532 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:36.532 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.532 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.532 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: Deploying daemon node-exporter.a on smithi005 2024-04-26T04:44:36.532 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:36.532 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: Deploying daemon node-exporter.a on smithi005 2024-04-26T04:44:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:37.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=infra.usagestats.collector t=2024-04-26T04:44:36.906237675Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-26T04:44:37.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=provisioning.datasources t=2024-04-26T04:44:36.90652364Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-26T04:44:37.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=provisioning.datasources t=2024-04-26T04:44:36.939368803Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T04:44:37.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=provisioning.datasources t=2024-04-26T04:44:36.970626836Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-26T04:44:37.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=provisioning.alerting t=2024-04-26T04:44:36.992520102Z level=info msg="starting to provision alerting" 2024-04-26T04:44:37.269 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=provisioning.alerting t=2024-04-26T04:44:36.992539847Z level=info msg="finished to provision alerting" 2024-04-26T04:44:37.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=ngalert.state.manager t=2024-04-26T04:44:36.992613426Z level=info msg="Warming state cache for startup" 2024-04-26T04:44:37.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=grafanaStorageLogger t=2024-04-26T04:44:36.992706967Z level=info msg="storage starting" 2024-04-26T04:44:37.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:36 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=http.server t=2024-04-26T04:44:36.996234762Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T04:44:37.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:37 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=ngalert.state.manager t=2024-04-26T04:44:37.002962859Z level=info msg="State cache has been initialized" states=0 duration=10.3469ms 2024-04-26T04:44:37.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:37 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=sqlstore.transactions t=2024-04-26T04:44:37.003036078Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:44:37.270 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:37 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=ngalert.multiorg.alertmanager t=2024-04-26T04:44:37.003041268Z level=info msg="starting MultiOrg Alertmanager" 2024-04-26T04:44:37.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:37 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=ticker t=2024-04-26T04:44:37.003178328Z level=info msg=starting first_tick=2024-04-26T04:44:40Z 2024-04-26T04:44:37.271 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:44:37 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[59666]: logger=sqlstore.transactions t=2024-04-26T04:44:37.057510884Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:44:38.275 INFO:teuthology.orchestra.run.smithi005.stdout:111669149729 2024-04-26T04:44:38.276 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:44:38.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:38 smithi005 ceph-mon[26371]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:38 smithi005 ceph-mon[36961]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:38 smithi111 ceph-mon[30255]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:39.030 INFO:teuthology.orchestra.run.smithi005.stdout:133143986203 2024-04-26T04:44:39.030 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:44:39.667 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:39 smithi005 bash[68058]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:44:40.221 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:39 smithi005 bash[68058]: Getting image source signatures 2024-04-26T04:44:40.222 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:39 smithi005 bash[68058]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:44:40.222 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:39 smithi005 bash[68058]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:44:40.222 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:39 smithi005 bash[68058]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:44:40.930 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:40 smithi005 ceph-mon[26371]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:40.930 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:40 smithi005 ceph-mon[36961]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:40 smithi111 ceph-mon[30255]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:42.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:44:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:42.479 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:42 smithi005 bash[68058]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:44:42.480 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:42 smithi005 bash[68058]: Writing manifest to image destination 2024-04-26T04:44:42.801 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:42 smithi005 ceph-mon[26371]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:42.802 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:42 smithi005 ceph-mon[36961]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:42.860 INFO:teuthology.orchestra.run.smithi005.stdout:167503724569 2024-04-26T04:44:42.861 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:44:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:42 smithi111 ceph-mon[30255]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:43.229 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:42 smithi005 podman[68058]: 2024-04-26 04:44:42.858690207 +0000 UTC m=+3.794472841 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:44:43.314 INFO:teuthology.orchestra.run.smithi005.stdout:77309411368 2024-04-26T04:44:43.314 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:44:43.612 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 podman[68058]: 2024-04-26 04:44:43.408497598 +0000 UTC m=+4.344280227 container create 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:43.612 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 podman[68058]: 2024-04-26 04:44:43.611276613 +0000 UTC m=+4.547059235 container init 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:43.946 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 podman[68058]: 2024-04-26 04:44:43.614345392 +0000 UTC m=+4.550128022 container start 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:43.946 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.618Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:44:43.946 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.618Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:44:43.946 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:44:43.947 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:44:43.948 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.619Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:44:43.949 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:44:43.950 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:44:43.951 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:44:43.951 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:44:43.951 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[68400]: ts=2024-04-26T04:44:43.620Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:44:43.951 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 bash[68058]: 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 2024-04-26T04:44:43.951 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:44:43 smithi005 systemd[1]: Started Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:44.466 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:44 smithi111 systemd[1]: Starting Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:44.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:44 smithi111 ceph-mon[30255]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:44.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:44 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:44 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:44 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.768 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:44 smithi111 bash[59923]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:44:44.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[26371]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[36961]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:44 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:45.036 INFO:teuthology.orchestra.run.smithi005.stdout:201863462932 2024-04-26T04:44:45.037 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:44:45.268 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:45 smithi111 bash[59923]: Getting image source signatures 2024-04-26T04:44:45.268 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:45 smithi111 bash[59923]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:44:45.268 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:45 smithi111 bash[59923]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:44:45.268 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:45 smithi111 bash[59923]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:44:45.294 INFO:teuthology.orchestra.run.smithi005.stdout:55834574892 2024-04-26T04:44:45.294 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:44:45.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:45 smithi005 ceph-mon[26371]: Deploying daemon node-exporter.b on smithi111 2024-04-26T04:44:45.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:45 smithi005 ceph-mon[36961]: Deploying daemon node-exporter.b on smithi111 2024-04-26T04:44:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:45 smithi111 ceph-mon[30255]: Deploying daemon node-exporter.b on smithi111 2024-04-26T04:44:46.128 INFO:teuthology.orchestra.run.smithi005.stdout:34359738419 2024-04-26T04:44:46.128 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:44:46.604 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:46.626 INFO:teuthology.orchestra.run.smithi005.stdout:236223201296 2024-04-26T04:44:46.627 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:44:46.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:46 smithi111 ceph-mon[30255]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:46.768 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:46 smithi111 bash[59923]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:44:46.768 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:46 smithi111 bash[59923]: Writing manifest to image destination 2024-04-26T04:44:46.890 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:46 smithi005 ceph-mon[26371]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:46.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:46 smithi005 ceph-mon[36961]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:47.065 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:46 smithi111 podman[59923]: 2024-04-26 04:44:46.739155043 +0000 UTC m=+2.172854702 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:44:47.065 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:46 smithi111 podman[59923]: 2024-04-26 04:44:46.846723934 +0000 UTC m=+2.280423622 container create 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:47.350 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:47.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 podman[59923]: 2024-04-26 04:44:47.065448001 +0000 UTC m=+2.499147647 container init 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:47.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 podman[59923]: 2024-04-26 04:44:47.068239584 +0000 UTC m=+2.501939237 container start 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:47.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.072Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.072Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.072Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.072Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:44:47.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:44:47.520 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:44:47.521 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.073Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:44:47.522 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[59976]: ts=2024-04-26T04:44:47.074Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 bash[59923]: 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 2024-04-26T04:44:47.523 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:44:47 smithi111 systemd[1]: Started Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:47.613 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:48.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[36961]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:48.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:48.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:48 smithi005 ceph-mon[26371]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:48 smithi111 ceph-mon[30255]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:49.709 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:49.710 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:49.710 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:49.713 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:49.713 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:44:50.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:50 smithi005 ceph-mon[26371]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:50.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:50 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:50.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:50 smithi005 ceph-mon[36961]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:50.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:50 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:50.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:50 smithi111 ceph-mon[30255]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:50.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:50 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:51.500 INFO:teuthology.orchestra.run.smithi005.stdout:111669149731 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3006305907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:51.693 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:51 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3006305907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:44:52.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:44:51 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:44:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:51 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:51 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:52.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:51 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:44:52.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:51 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3006305907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:44:52.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:52 smithi005 ceph-mon[26371]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:52.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:52 smithi005 ceph-mon[36961]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:52 smithi111 ceph-mon[30255]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:53.671 INFO:teuthology.orchestra.run.smithi005.stdout:133143986207 2024-04-26T04:44:54.821 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:54 smithi005 ceph-mon[26371]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:54.821 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:54 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/153852166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:44:54.822 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:54 smithi005 ceph-mon[36961]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:54.822 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:54 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/153852166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:44:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:54 smithi111 ceph-mon[30255]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:54 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/153852166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:44:55.430 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149729 got 111669149731 for osd.3 2024-04-26T04:44:55.431 DEBUG:teuthology.parallel:result is None 2024-04-26T04:44:55.959 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986207 for osd.4 2024-04-26T04:44:55.959 DEBUG:teuthology.parallel:result is None 2024-04-26T04:44:56.356 INFO:teuthology.orchestra.run.smithi005.stdout:34359738421 2024-04-26T04:44:56.801 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:56 smithi005 ceph-mon[26371]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:56.802 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:56 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4012361532' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:44:56.802 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:56 smithi005 ceph-mon[36961]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:56.802 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:56 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4012361532' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:44:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:56 smithi111 ceph-mon[30255]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:56 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4012361532' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:44:58.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:58 smithi005 ceph-mon[26371]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:58 smithi005 ceph-mon[36961]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.794 INFO:teuthology.orchestra.run.smithi005.stdout:55834574896 2024-04-26T04:44:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:58 smithi111 ceph-mon[30255]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:59.608 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738419 got 34359738421 for osd.0 2024-04-26T04:44:59.608 DEBUG:teuthology.parallel:result is None 2024-04-26T04:44:59.909 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:44:59 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1629488547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:44:59.909 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:44:59 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1629488547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:45:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:44:59 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1629488547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:45:00.559 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574896 for osd.1 2024-04-26T04:45:00.559 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:00.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:00 smithi111 ceph-mon[30255]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:00.947 INFO:teuthology.orchestra.run.smithi005.stdout:236223201299 2024-04-26T04:45:00.947 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:00 smithi005 ceph-mon[26371]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:00.947 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:00 smithi005 ceph-mon[36961]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:01.654 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:01 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1758170653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:45:01.654 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:01 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1758170653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:45:01.877 INFO:teuthology.orchestra.run.smithi005.stdout:77309411373 2024-04-26T04:45:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:01 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1758170653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:45:02.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:45:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:45:02.098 INFO:teuthology.orchestra.run.smithi005.stdout:201863462937 2024-04-26T04:45:02.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:02 smithi005 ceph-mon[26371]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:02.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:02 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/391404330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:45:02.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:02 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1874366159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:45:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:02 smithi005 ceph-mon[36961]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:02 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/391404330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:45:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:02 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1874366159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:45:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:02 smithi111 ceph-mon[30255]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:02 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/391404330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:45:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:02 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1874366159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:45:03.149 INFO:teuthology.orchestra.run.smithi005.stdout:167503724574 2024-04-26T04:45:03.601 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411373 for osd.2 2024-04-26T04:45:03.601 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:03.935 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:03 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1496053319' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:45:03.935 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:03 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1496053319' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:45:03.966 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462937 for osd.6 2024-04-26T04:45:03.966 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:03 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1496053319' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:45:04.421 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724574 for osd.5 2024-04-26T04:45:04.421 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:04.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:04 smithi005 ceph-mon[26371]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:04.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:04 smithi005 ceph-mon[36961]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:04.901 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201296 got 236223201299 for osd.7 2024-04-26T04:45:04.901 DEBUG:teuthology.parallel:result is None 2024-04-26T04:45:04.901 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:45:04.902 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:45:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:04 smithi111 ceph-mon[30255]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:05.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:05 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:05 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:05 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:06.516 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:06.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[36961]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:06.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:06.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:06.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:06.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:06.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:06.830 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[26371]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:06.830 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:06.830 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:06.830 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:06.830 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:06.830 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:06 smithi111 ceph-mon[30255]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:08.651 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:08 smithi005 ceph-mon[26371]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:45:08.651 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:08 smithi005 ceph-mon[26371]: Reconfiguring daemon alertmanager.a on smithi005 2024-04-26T04:45:08.651 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:08 smithi005 ceph-mon[26371]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:08.651 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:08 smithi005 ceph-mon[36961]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:45:08.651 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:08 smithi005 ceph-mon[36961]: Reconfiguring daemon alertmanager.a on smithi005 2024-04-26T04:45:08.651 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:08 smithi005 ceph-mon[36961]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:08.804 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:45:08.805 INFO:teuthology.orchestra.run.smithi005.stderr:dumped all 2024-04-26T04:45:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:08 smithi111 ceph-mon[30255]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:45:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:08 smithi111 ceph-mon[30255]: Reconfiguring daemon alertmanager.a on smithi005 2024-04-26T04:45:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:08 smithi111 ceph-mon[30255]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:09.778 INFO:teuthology.orchestra.run.smithi005.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-26T04:45:07.483180+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6708,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6868992,"data_stored":3897850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001993"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.767459+0000","last_change":"2024-04-26T04:44:08.479180+0000","last_active":"2024-04-26T04:44:09.767459+0000","last_peered":"2024-04-26T04:44:09.767459+0000","last_clean":"2024-04-26T04:44:09.767459+0000","last_became_active":"2024-04-26T04:44:08.478822+0000","last_became_peered":"2024-04-26T04:44:08.478822+0000","last_unstale":"2024-04-26T04:44:09.767459+0000","last_undegraded":"2024-04-26T04:44:09.767459+0000","last_fullsized":"2024-04-26T04:44:09.767459+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:08:10.312422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.713038+0000","last_change":"2024-04-26T04:44:08.965639+0000","last_active":"2024-04-26T04:44:09.713038+0000","last_peered":"2024-04-26T04:44:09.713038+0000","last_clean":"2024-04-26T04:44:09.713038+0000","last_became_active":"2024-04-26T04:44:08.965296+0000","last_became_peered":"2024-04-26T04:44:08.965296+0000","last_unstale":"2024-04-26T04:44:09.713038+0000","last_undegraded":"2024-04-26T04:44:09.713038+0000","last_fullsized":"2024-04-26T04:44:09.713038+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:17:21.026489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.481247+0000","last_change":"2024-04-26T04:44:08.965486+0000","last_active":"2024-04-26T04:44:09.481247+0000","last_peered":"2024-04-26T04:44:09.481247+0000","last_clean":"2024-04-26T04:44:09.481247+0000","last_became_active":"2024-04-26T04:44:08.965051+0000","last_became_peered":"2024-04-26T04:44:08.965051+0000","last_unstale":"2024-04-26T04:44:09.481247+0000","last_undegraded":"2024-04-26T04:44:09.481247+0000","last_fullsized":"2024-04-26T04:44:09.481247+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:47:18.277374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.481225+0000","last_change":"2024-04-26T04:44:08.481145+0000","last_active":"2024-04-26T04:44:09.481225+0000","last_peered":"2024-04-26T04:44:09.481225+0000","last_clean":"2024-04-26T04:44:09.481225+0000","last_became_active":"2024-04-26T04:44:08.480765+0000","last_became_peered":"2024-04-26T04:44:08.480765+0000","last_unstale":"2024-04-26T04:44:09.481225+0000","last_undegraded":"2024-04-26T04:44:09.481225+0000","last_fullsized":"2024-04-26T04:44:09.481225+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:48:08.184965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.481223+0000","last_change":"2024-04-26T04:43:35.916221+0000","last_active":"2024-04-26T04:44:09.481223+0000","last_peered":"2024-04-26T04:44:09.481223+0000","last_clean":"2024-04-26T04:44:09.481223+0000","last_became_active":"2024-04-26T04:43:35.915728+0000","last_became_peered":"2024-04-26T04:43:35.915728+0000","last_unstale":"2024-04-26T04:44:09.481223+0000","last_undegraded":"2024-04-26T04:44:09.481223+0000","last_fullsized":"2024-04-26T04:44:09.481223+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_clean_scrub_stamp":"2024-04-26T04:41:33.492763+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:47:26.279649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.767395+0000","last_change":"2024-04-26T04:44:08.479414+0000","last_active":"2024-04-26T04:44:09.767395+0000","last_peered":"2024-04-26T04:44:09.767395+0000","last_clean":"2024-04-26T04:44:09.767395+0000","last_became_active":"2024-04-26T04:44:08.478584+0000","last_became_peered":"2024-04-26T04:44:08.478584+0000","last_unstale":"2024-04-26T04:44:09.767395+0000","last_undegraded":"2024-04-26T04:44:09.767395+0000","last_fullsized":"2024-04-26T04:44:09.767395+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:02:29.012874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.480913+0000","last_change":"2024-04-26T04:44:09.231105+0000","last_active":"2024-04-26T04:44:09.480913+0000","last_peered":"2024-04-26T04:44:09.480913+0000","last_clean":"2024-04-26T04:44:09.480913+0000","last_became_active":"2024-04-26T04:44:09.230737+0000","last_became_peered":"2024-04-26T04:44:09.230737+0000","last_unstale":"2024-04-26T04:44:09.480913+0000","last_undegraded":"2024-04-26T04:44:09.480913+0000","last_fullsized":"2024-04-26T04:44:09.480913+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:25:04.219657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.767326+0000","last_change":"2024-04-26T04:44:08.477772+0000","last_active":"2024-04-26T04:44:09.767326+0000","last_peered":"2024-04-26T04:44:09.767326+0000","last_clean":"2024-04-26T04:44:09.767326+0000","last_became_active":"2024-04-26T04:44:08.477431+0000","last_became_peered":"2024-04-26T04:44:08.477431+0000","last_unstale":"2024-04-26T04:44:09.767326+0000","last_undegraded":"2024-04-26T04:44:09.767326+0000","last_fullsized":"2024-04-26T04:44:09.767326+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:50:07.493196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.713034+0000","last_change":"2024-04-26T04:44:08.480906+0000","last_active":"2024-04-26T04:44:09.713034+0000","last_peered":"2024-04-26T04:44:09.713034+0000","last_clean":"2024-04-26T04:44:09.713034+0000","last_became_active":"2024-04-26T04:44:08.480613+0000","last_became_peered":"2024-04-26T04:44:08.480613+0000","last_unstale":"2024-04-26T04:44:09.713034+0000","last_undegraded":"2024-04-26T04:44:09.713034+0000","last_fullsized":"2024-04-26T04:44:09.713034+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:20:31.237343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.56299999999999994}]},{"osd":1,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.63300000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52200000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.64700000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.54300000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.67600000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.66100000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":855880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.216,"5min":1.216,"15min":1.216},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":16.103999999999999,"5min":16.103999999999999,"15min":16.103999999999999},"last":0.59399999999999997}]},{"osd":1,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":9.75,"5min":9.75,"15min":9.75},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.61699999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":9.7249999999999996,"5min":9.7249999999999996,"15min":9.7249999999999996},"last":0.64300000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.226,"5min":1.226,"15min":1.226},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":16.128,"5min":16.128,"15min":16.128},"last":0.54900000000000004}]},{"osd":4,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.63}]},{"osd":5,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.69099999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:44:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.66800000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:18 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34599999999999997,"15min":0.32100000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.61299999999999999,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.37,"15min":0.33900000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.59899999999999998,"5min":0.627,"15min":0.627},"last":0.46999999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:44:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.28299999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60699999999999998,"5min":0.63,"15min":0.63},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38600000000000001,"15min":0.38},"min":{"1min":0.27900000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.48999999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:45:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.50700000000000001,"15min":0.51100000000000001},"min":{"1min":0.375,"5min":0.315,"15min":0.315},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.46200000000000002,"15min":0.46999999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.58199999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.58199999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:44:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58199999999999996,"15min":0.57899999999999996},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.625}]},{"osd":5,"last update":"Fri Apr 26 04:44:44 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60699999999999998,"15min":0.61199999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80300000000000005,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.63800000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:44:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.69099999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:44:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.78300000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.35099999999999998},"min":{"1min":0.28699999999999998,"5min":0.192,"15min":0.192},"max":{"1min":0.53600000000000003,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.378},{"interface":"front","average":{"1min":0.442,"5min":0.373,"15min":0.34200000000000003},"min":{"1min":0.308,"5min":0.214,"15min":0.214},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.27900000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41299999999999998},"min":{"1min":0.29399999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.39800000000000002},"min":{"1min":0.31,"5min":0.248,"15min":0.248},"max":{"1min":0.65600000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.35699999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:45:05 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.499,"15min":0.498},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.501,"15min":0.51000000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.67400000000000004,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.40100000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:44:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.746},{"interface":"front","average":{"1min":0.625,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.59299999999999997}]},{"osd":5,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.622,"15min":0.623},"min":{"1min":0.42399999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.64300000000000002,"15min":0.64600000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.70499999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:44:09 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.58199999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.60799999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:41 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38800000000000001,"15min":0.39000000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53800000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.27800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.58199999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:44:41 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.41199999999999998,"15min":0.41099999999999998},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.498},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.33000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.53100000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:44:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.48499999999999999,"15min":0.47699999999999998},"min":{"1min":0.41799999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.49399999999999999,"15min":0.49099999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.54900000000000004}]},{"osd":4,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71399999999999997,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.67400000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:44:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.63300000000000001,"15min":0.64500000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.80100000000000005,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76200000000000001,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68200000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.752},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.71299999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:44:35 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.69699999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.40200000000000002,"15min":0.39500000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.434,"5min":0.441,"15min":0.439},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.119,"5min":1.119,"15min":1.119},"last":0.32600000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:45:00 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.443,"15min":0.44900000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.316}]},{"osd":2,"last update":"Fri Apr 26 04:45:00 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46999999999999997,"15min":0.46400000000000002},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.45400000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.68899999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.75,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56999999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:44:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.46300000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.77200000000000002,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.47399999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58199999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.499}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.46999999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.65600000000000003,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.443},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.41299999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.40400000000000003}]},{"osd":1,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.52000000000000002,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.67500000000000004,"5min":0.753,"15min":0.753},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.53700000000000003,"15min":0.53400000000000003},"min":{"1min":0.40600000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.436}]},{"osd":2,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55300000000000005,"15min":0.54600000000000004},"min":{"1min":0.45500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77400000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.376},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.48699999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53600000000000003,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55700000000000005,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47299999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:44:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.40000000000000002,"15min":0.38100000000000001},"min":{"1min":0.32300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.40000000000000002,"15min":0.38400000000000001},"min":{"1min":0.311,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.46000000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:44:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.41099999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:44:33 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.41999999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.375,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.69699999999999995}]},{"osd":1,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.61599999999999999,"15min":0.56499999999999995},"min":{"1min":0.38400000000000001,"5min":0.374,"15min":0.374},"max":{"1min":8.2089999999999996,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54900000000000004,"15min":0.54700000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.57999999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.56899999999999995,"15min":0.56499999999999995},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.66400000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.626,"15min":0.57499999999999996},"min":{"1min":0.41999999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.1720000000000006,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55400000000000005,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.73899999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.501},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.442},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.46500000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:44:09 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.48499999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.44}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:45:09.780 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:45:10.844 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:10 smithi005 ceph-mon[36961]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:45:10.845 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:10 smithi005 ceph-mon[36961]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:10.845 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:10 smithi005 ceph-mon[26371]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:45:10.845 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:10 smithi005 ceph-mon[26371]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:10 smithi111 ceph-mon[30255]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:45:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:10 smithi111 ceph-mon[30255]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:11.793 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:11.918 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:11 smithi005 systemd[1]: Stopping Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:12.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:45:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:45:12.908 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:12 smithi005 ceph-mon[36961]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:12.908 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:12 smithi005 ceph-mon[26371]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:13.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:12 smithi111 ceph-mon[30255]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:13.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[66571]: ts=2024-04-26T04:45:13.061Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:45:13.786 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:45:13.787 INFO:teuthology.orchestra.run.smithi005.stderr:dumped all 2024-04-26T04:45:13.979 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:13 smithi005 podman[70488]: 2024-04-26 04:45:13.621735963 +0000 UTC m=+1.096661619 container died f28e964e37c3c7199660ab3d686753951db43b68f8b74e78187686f38b1bb998 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:14.377 INFO:teuthology.orchestra.run.smithi005.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-04-26T04:45:13.484352+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6708,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6868992,"data_stored":3897850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002087"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.767459+0000","last_change":"2024-04-26T04:44:08.479180+0000","last_active":"2024-04-26T04:44:09.767459+0000","last_peered":"2024-04-26T04:44:09.767459+0000","last_clean":"2024-04-26T04:44:09.767459+0000","last_became_active":"2024-04-26T04:44:08.478822+0000","last_became_peered":"2024-04-26T04:44:08.478822+0000","last_unstale":"2024-04-26T04:44:09.767459+0000","last_undegraded":"2024-04-26T04:44:09.767459+0000","last_fullsized":"2024-04-26T04:44:09.767459+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:08:10.312422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.713038+0000","last_change":"2024-04-26T04:44:08.965639+0000","last_active":"2024-04-26T04:44:09.713038+0000","last_peered":"2024-04-26T04:44:09.713038+0000","last_clean":"2024-04-26T04:44:09.713038+0000","last_became_active":"2024-04-26T04:44:08.965296+0000","last_became_peered":"2024-04-26T04:44:08.965296+0000","last_unstale":"2024-04-26T04:44:09.713038+0000","last_undegraded":"2024-04-26T04:44:09.713038+0000","last_fullsized":"2024-04-26T04:44:09.713038+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:17:21.026489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.481247+0000","last_change":"2024-04-26T04:44:08.965486+0000","last_active":"2024-04-26T04:44:09.481247+0000","last_peered":"2024-04-26T04:44:09.481247+0000","last_clean":"2024-04-26T04:44:09.481247+0000","last_became_active":"2024-04-26T04:44:08.965051+0000","last_became_peered":"2024-04-26T04:44:08.965051+0000","last_unstale":"2024-04-26T04:44:09.481247+0000","last_undegraded":"2024-04-26T04:44:09.481247+0000","last_fullsized":"2024-04-26T04:44:09.481247+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:47:18.277374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.481225+0000","last_change":"2024-04-26T04:44:08.481145+0000","last_active":"2024-04-26T04:44:09.481225+0000","last_peered":"2024-04-26T04:44:09.481225+0000","last_clean":"2024-04-26T04:44:09.481225+0000","last_became_active":"2024-04-26T04:44:08.480765+0000","last_became_peered":"2024-04-26T04:44:08.480765+0000","last_unstale":"2024-04-26T04:44:09.481225+0000","last_undegraded":"2024-04-26T04:44:09.481225+0000","last_fullsized":"2024-04-26T04:44:09.481225+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:48:08.184965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.481223+0000","last_change":"2024-04-26T04:43:35.916221+0000","last_active":"2024-04-26T04:44:09.481223+0000","last_peered":"2024-04-26T04:44:09.481223+0000","last_clean":"2024-04-26T04:44:09.481223+0000","last_became_active":"2024-04-26T04:43:35.915728+0000","last_became_peered":"2024-04-26T04:43:35.915728+0000","last_unstale":"2024-04-26T04:44:09.481223+0000","last_undegraded":"2024-04-26T04:44:09.481223+0000","last_fullsized":"2024-04-26T04:44:09.481223+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_clean_scrub_stamp":"2024-04-26T04:41:33.492763+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:47:26.279649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.767395+0000","last_change":"2024-04-26T04:44:08.479414+0000","last_active":"2024-04-26T04:44:09.767395+0000","last_peered":"2024-04-26T04:44:09.767395+0000","last_clean":"2024-04-26T04:44:09.767395+0000","last_became_active":"2024-04-26T04:44:08.478584+0000","last_became_peered":"2024-04-26T04:44:08.478584+0000","last_unstale":"2024-04-26T04:44:09.767395+0000","last_undegraded":"2024-04-26T04:44:09.767395+0000","last_fullsized":"2024-04-26T04:44:09.767395+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:02:29.012874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.480913+0000","last_change":"2024-04-26T04:44:09.231105+0000","last_active":"2024-04-26T04:44:09.480913+0000","last_peered":"2024-04-26T04:44:09.480913+0000","last_clean":"2024-04-26T04:44:09.480913+0000","last_became_active":"2024-04-26T04:44:09.230737+0000","last_became_peered":"2024-04-26T04:44:09.230737+0000","last_unstale":"2024-04-26T04:44:09.480913+0000","last_undegraded":"2024-04-26T04:44:09.480913+0000","last_fullsized":"2024-04-26T04:44:09.480913+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:25:04.219657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.767326+0000","last_change":"2024-04-26T04:44:08.477772+0000","last_active":"2024-04-26T04:44:09.767326+0000","last_peered":"2024-04-26T04:44:09.767326+0000","last_clean":"2024-04-26T04:44:09.767326+0000","last_became_active":"2024-04-26T04:44:08.477431+0000","last_became_peered":"2024-04-26T04:44:08.477431+0000","last_unstale":"2024-04-26T04:44:09.767326+0000","last_undegraded":"2024-04-26T04:44:09.767326+0000","last_fullsized":"2024-04-26T04:44:09.767326+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:50:07.493196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:44:09.713034+0000","last_change":"2024-04-26T04:44:08.480906+0000","last_active":"2024-04-26T04:44:09.713034+0000","last_peered":"2024-04-26T04:44:09.713034+0000","last_clean":"2024-04-26T04:44:09.713034+0000","last_became_active":"2024-04-26T04:44:08.480613+0000","last_became_peered":"2024-04-26T04:44:08.480613+0000","last_unstale":"2024-04-26T04:44:09.713034+0000","last_undegraded":"2024-04-26T04:44:09.713034+0000","last_fullsized":"2024-04-26T04:44:09.713034+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:20:31.237343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.64000000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.51600000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.58199999999999996}]},{"osd":3,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.61199999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.623},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.68000000000000005}]},{"osd":5,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.66200000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.70199999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":855880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73399999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.51900000000000002,"5min":1.0760000000000001,"15min":1.169},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.752,"5min":16.103999999999999,"15min":16.103999999999999},"last":0.63200000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.89500000000000002,"15min":0.94599999999999995},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.79400000000000004,"5min":9.75,"15min":9.75},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.53700000000000003},"min":{"1min":0.39500000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79200000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.64900000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.52100000000000002,"15min":0.51900000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.84899999999999998,"15min":0.90100000000000002},"min":{"1min":0.432,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79000000000000004,"5min":9.7249999999999996,"15min":9.7249999999999996},"last":0.54100000000000004}]},{"osd":3,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52800000000000002,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68999999999999995,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.0920000000000001,"15min":1.181},"min":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77000000000000002,"5min":16.128,"15min":16.128},"last":0.55500000000000005}]},{"osd":4,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.317,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.435,"15min":0.42999999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.45400000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.434,"15min":0.42999999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.47599999999999998,"15min":0.47399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51800000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:44:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.435}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:18 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34599999999999997,"15min":0.32100000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.61299999999999999,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.434,"5min":0.37,"15min":0.33900000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.59899999999999998,"5min":0.627,"15min":0.627},"last":0.28699999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:44:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.28299999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60699999999999998,"5min":0.63,"15min":0.63},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38600000000000001,"15min":0.38},"min":{"1min":0.27900000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36899999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:45:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.50700000000000001,"15min":0.51100000000000001},"min":{"1min":0.375,"5min":0.315,"15min":0.315},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.46200000000000002,"15min":0.46999999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.58199999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.32100000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:44:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58199999999999996,"15min":0.57899999999999996},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.67800000000000005}]},{"osd":5,"last update":"Fri Apr 26 04:44:44 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60699999999999998,"15min":0.61199999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80300000000000005,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.71599999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:44:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.621},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.64400000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:44:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.61099999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738425,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.35099999999999998},"min":{"1min":0.28699999999999998,"5min":0.192,"15min":0.192},"max":{"1min":0.53600000000000003,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.433},{"interface":"front","average":{"1min":0.442,"5min":0.373,"15min":0.34200000000000003},"min":{"1min":0.308,"5min":0.214,"15min":0.214},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.27400000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41299999999999998},"min":{"1min":0.29399999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.39800000000000002},"min":{"1min":0.31,"5min":0.248,"15min":0.248},"max":{"1min":0.65600000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.31}]},{"osd":3,"last update":"Fri Apr 26 04:45:05 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.499,"15min":0.498},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.501,"15min":0.51000000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.67400000000000004,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.39700000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:44:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.625,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.67700000000000005}]},{"osd":5,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.622,"15min":0.623},"min":{"1min":0.42399999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.64300000000000002,"15min":0.64600000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.69399999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:44:09 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.60099999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:44:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.62}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:41 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38800000000000001,"15min":0.39000000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53800000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.22500000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.27800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.255}]},{"osd":1,"last update":"Fri Apr 26 04:44:41 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.41199999999999998,"15min":0.41099999999999998},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.33000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.29099999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.48399999999999999,"15min":0.47599999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66000000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.48599999999999999,"15min":0.48799999999999999},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.61599999999999999,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.378}]},{"osd":4,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71399999999999997,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41999999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:44:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.63300000000000001,"15min":0.64500000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.80100000000000005,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76200000000000001,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.52200000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.435,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76400000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.435},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64100000000000001,"15min":0.64900000000000002},"min":{"1min":0.42499999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.48599999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:44:35 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.55800000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.40200000000000002,"15min":0.39500000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.441,"15min":0.439},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.119,"5min":1.119,"15min":1.119},"last":0.499}]},{"osd":1,"last update":"Fri Apr 26 04:45:00 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.443,"15min":0.44900000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.47999999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:45:00 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46999999999999997,"15min":0.46400000000000002},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.52200000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:44:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.68899999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.75,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.73999999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:44:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.755},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.46300000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.77200000000000002,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.69599999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:44:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.79300000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.70999999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.46999999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.65600000000000003,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.41299999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.64000000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.52000000000000002,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.67500000000000004,"5min":0.753,"15min":0.753},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.53700000000000003,"15min":0.53400000000000003},"min":{"1min":0.40600000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.66800000000000004}]},{"osd":2,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55300000000000005,"15min":0.54600000000000004},"min":{"1min":0.45500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77400000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.54500000000000004}]},{"osd":3,"last update":"Fri Apr 26 04:44:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53600000000000003,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55700000000000005,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65600000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:44:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.40000000000000002,"15min":0.38100000000000001},"min":{"1min":0.32300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.40000000000000002,"15min":0.38400000000000001},"min":{"1min":0.311,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.46999999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:45:10 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.42499999999999999,"15min":0.41299999999999998},"min":{"1min":0.32300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.42899999999999999,"15min":0.41199999999999998},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.44800000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:44:33 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.378}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.375,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.57699999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.61599999999999999,"15min":0.56499999999999995},"min":{"1min":0.38400000000000001,"5min":0.374,"15min":0.374},"max":{"1min":8.2089999999999996,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54900000000000004,"15min":0.54700000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.501}]},{"osd":2,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.56899999999999995,"15min":0.56499999999999995},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.60299999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.626,"15min":0.57499999999999996},"min":{"1min":0.41999999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.1720000000000006,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55400000000000005,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.626}]},{"osd":4,"last update":"Fri Apr 26 04:44:45 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.442},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.40999999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:44:09 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.374}]},{"osd":7,"last update":"Fri Apr 26 04:44:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.433}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:45:14.379 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:45:14.379 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:45:14.380 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:45:14.380 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:14 smithi005 ceph-mon[36961]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:14.620 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:14 smithi005 ceph-mon[26371]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:14.946 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:14 smithi005 podman[70488]: 2024-04-26 04:45:14.61924526 +0000 UTC m=+2.094170916 container cleanup f28e964e37c3c7199660ab3d686753951db43b68f8b74e78187686f38b1bb998 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:14.947 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:14 smithi005 bash[70488]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:45:14.947 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:14 smithi005 podman[70526]: 2024-04-26 04:45:14.845630727 +0000 UTC m=+1.750610213 container remove f28e964e37c3c7199660ab3d686753951db43b68f8b74e78187686f38b1bb998 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:14.947 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:14 smithi005 podman[70526]: 2024-04-26 04:45:14.946330284 +0000 UTC m=+1.851309726 volume remove e500cc40f9cad8f9feac1ea0e659547d28ba5445b66c17daff1250f4fdc068fe 2024-04-26T04:45:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:14 smithi111 ceph-mon[30255]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:15.082 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:15.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:15 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:45:15.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:15 smithi005 systemd[1]: Stopped Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:15.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:15 smithi005 systemd[1]: Starting Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:15.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:15 smithi005 ceph-mon[26371]: from='client.24518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:45:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:15 smithi005 ceph-mon[36961]: from='client.24518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:45:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:15 smithi111 ceph-mon[30255]: from='client.24518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:45:16.724 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:16 smithi005 ceph-mon[26371]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:16.725 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:16 smithi005 ceph-mon[36961]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:16.841 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:45:16.841 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:45:16.980 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:16 smithi005 podman[70804]: 2024-04-26 04:45:16.72392899 +0000 UTC m=+0.108105124 volume create bda220482b45896b851925ff39aa4aca40558f4786c8a3226c64c4eba7097cd4 2024-04-26T04:45:16.980 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:16 smithi005 podman[70804]: 2024-04-26 04:45:16.634043377 +0000 UTC m=+0.018219508 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:45:16.980 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:16 smithi005 podman[70804]: 2024-04-26 04:45:16.849456228 +0000 UTC m=+0.233632358 container create 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:16 smithi111 ceph-mon[30255]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:17.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/576064940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:45:17.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/576064940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:45:17.808 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:45:17.809 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T04:45:17.809 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T04:45:17.818 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi005.front.sepia.ceph.com 2024-04-26T04:45:17.819 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-26T04:45:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:17 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/576064940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:45:18.107 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 podman[70804]: 2024-04-26 04:45:17.915516938 +0000 UTC m=+1.299693071 container init 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 podman[70804]: 2024-04-26 04:45:17.918473087 +0000 UTC m=+1.302649220 container start 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.933Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.933Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.934Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.5 port=9094 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.935Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.963Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:45:18.230 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.964Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:45:18.231 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.966Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:45:18.231 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:17 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:17.966Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:45:18.231 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:18 smithi005 bash[70804]: 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b 2024-04-26T04:45:18.231 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:18 smithi005 systemd[1]: Started Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:18.738 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:18 smithi111 ceph-mon[30255]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:18.738 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:18 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:18.738 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:18 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:18.948 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:18 smithi005 ceph-mon[36961]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:18.948 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:18 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:18.948 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:18 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:18.948 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:18 smithi005 ceph-mon[26371]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:18.948 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:18 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:18.948 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:18 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:19.819 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:19 smithi111 ceph-mon[30255]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:45:19.820 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:19 smithi111 ceph-mon[30255]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:45:19.938 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:19 smithi005 ceph-mon[26371]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:45:19.938 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:19 smithi005 ceph-mon[26371]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:45:19.938 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:19 smithi005 ceph-mon[36961]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:45:19.938 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:19 smithi005 ceph-mon[36961]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:45:20.229 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:19.936Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000998129s 2024-04-26T04:45:20.819 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:20 smithi111 ceph-mon[30255]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:20.819 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:20 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:45:20.819 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:20 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:20.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:20 smithi005 ceph-mon[26371]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:20.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:20 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:45:20.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:20 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:20.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:20 smithi005 ceph-mon[36961]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:20.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:20 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:45:20.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:20 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:21.215 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:20 smithi111 systemd[1]: Stopping Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:21.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:45:21.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:45:21.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:45:21.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:45:21.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.214Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.215Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[57618]: ts=2024-04-26T04:45:21.215Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:45:21.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 podman[60753]: 2024-04-26 04:45:21.224514607 +0000 UTC m=+0.113818708 container died f1be29f4d04dde61281a9061ff65a5fd4c8bcb5a1441830701a55d5c3385e889 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:21.907 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 podman[60753]: 2024-04-26 04:45:21.597536619 +0000 UTC m=+0.486840727 container cleanup f1be29f4d04dde61281a9061ff65a5fd4c8bcb5a1441830701a55d5c3385e889 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:21.907 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 bash[60753]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:45:21.908 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-mon[30255]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:21.908 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:45:21.908 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-mon[30255]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:45:21.908 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[26371]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[26371]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[26367]: 2024-04-26T04:45:21.622+0000 7f74360c7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[36961]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:45:21.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[36961]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:45:21.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:45:22.240 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:21 smithi111 podman[60767]: 2024-04-26 04:45:21.907152555 +0000 UTC m=+0.685203349 container remove f1be29f4d04dde61281a9061ff65a5fd4c8bcb5a1441830701a55d5c3385e889 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:22.241 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:45:22.241 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 systemd[1]: Stopped Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:22.241 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.036s CPU time. 2024-04-26T04:45:22.241 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 systemd[1]: Starting Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:22.392 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 podman[60861]: 2024-04-26 04:45:22.143838462 +0000 UTC m=+0.018508737 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 podman[60861]: 2024-04-26 04:45:22.258215833 +0000 UTC m=+0.132886100 container create f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 podman[60861]: 2024-04-26 04:45:22.452366541 +0000 UTC m=+0.327036807 container init f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 podman[60861]: 2024-04-26 04:45:22.454901995 +0000 UTC m=+0.329572270 container start f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.479Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.479Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:45:22.502 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.479Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi111 (none))" 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.479Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.479Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.480Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.481Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.483Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.483Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.484Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.484Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.877µs 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.484Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.484Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.485Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.485Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.368µs wal_replay_duration=613.564µs wbl_replay_duration=100ns total_replay_duration=653.163µs 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.485Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:45:22.503 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.486Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:45:22.504 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.486Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:45:22.504 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.501Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.260776ms db_storage=786ns remote_storage=1.056µs web_handler=280ns query_engine=566ns scrape=561.112µs scrape_sd=27.564µs notify=18.08µs notify_sd=5.535µs rules=14.370041ms tracing=6.587µs 2024-04-26T04:45:22.504 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.501Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:45:22.504 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:45:22.501Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:45:22.682 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:22.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 bash[60861]: f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 2024-04-26T04:45:22.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:45:22 smithi111 systemd[1]: Started Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: pgmap v46: 41 pgs: 26 creating+peering, 9 active+clean, 6 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:45:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: fsmap foo:0 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: Saving service mds.foo spec with placement count:2 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005:9093"}]: dispatch 2024-04-26T04:45:22.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:22 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: pgmap v46: 41 pgs: 26 creating+peering, 9 active+clean, 6 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:45:22.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: fsmap foo:0 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: Saving service mds.foo spec with placement count:2 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005:9093"}]: dispatch 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: pgmap v46: 41 pgs: 26 creating+peering, 9 active+clean, 6 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:45:22.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: fsmap foo:0 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: Saving service mds.foo spec with placement count:2 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:45:22.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005:9093"}]: dispatch 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STOPPING 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STOPPED 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STARTING 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Serving on http://:::9283 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STARTED 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STOPPING 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STOPPED 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STARTING 2024-04-26T04:45:22.953 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Serving on http://:::9283 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STARTED 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STOPPING 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STOPPED 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STARTING 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Serving on http://:::9283 2024-04-26T04:45:22.954 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:45:22] ENGINE Bus STARTED 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005:9093"}]: dispatch 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:45:23.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005:9093"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:23.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:23.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:23 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:24.004 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005:9093"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:45:24.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:24.006 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:23 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:24.948 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:24 smithi111 ceph-mon[30255]: pgmap v50: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:24.948 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:24 smithi111 ceph-mon[30255]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:45:24.948 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:24 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:45:24.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:24 smithi005 ceph-mon[26371]: pgmap v50: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:24.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:24 smithi005 ceph-mon[26371]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:45:24.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:24 smithi005 ceph-mon[26371]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:45:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:24 smithi005 ceph-mon[36961]: pgmap v50: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:24 smithi005 ceph-mon[36961]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:45:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:24 smithi005 ceph-mon[36961]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:45:25.479 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T04:45:25.817 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:26.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:26 smithi005 ceph-mon[26371]: pgmap v52: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:26.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:26 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:26.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:26 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:26.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:26 smithi005 ceph-mon[36961]: pgmap v52: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:26.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:26 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:26.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:26 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:26 smithi111 ceph-mon[30255]: pgmap v52: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:26 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:26 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:28.229 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:45:27 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:45:27.938Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003082269s 2024-04-26T04:45:28.515 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:45:28.525 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:45:28.526 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:45:28.526 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:45:28.915 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:28.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:28 smithi005 ceph-mon[26371]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:28.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:28 smithi005 ceph-mon[36961]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:28 smithi111 ceph-mon[30255]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:45:30.928 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:45:30.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:45:31.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:45:31.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:31.576 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:45:31.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:31 smithi005 ceph-mon[26371]: Deploying daemon mds.foo.smithi005.zdnlsb on smithi005 2024-04-26T04:45:31.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:31 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/904117970' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:45:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:31 smithi005 ceph-mon[36961]: Deploying daemon mds.foo.smithi005.zdnlsb on smithi005 2024-04-26T04:45:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:31 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/904117970' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:45:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:31 smithi111 ceph-mon[30255]: Deploying daemon mds.foo.smithi005.zdnlsb on smithi005 2024-04-26T04:45:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:31 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/904117970' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:45:32.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:45:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:45:32.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:32 smithi005 ceph-mon[26371]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:32 smithi005 ceph-mon[36961]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.790 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":21,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6800","nonce":1913329511},{"type":"v1","addr":"172.21.15.5:6801","nonce":1913329511}]},"active_addr":"172.21.15.5:6801/1913329511","active_change":"2024-04-26T04:44:05.461138+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24373,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.5:8443/","prometheus":"http://172.21.15.5:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":4008904229}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":2166298037}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":188422345}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":4211393178}]}]} 2024-04-26T04:45:32.795 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:45:32.795 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:45:32.795 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:45:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:32 smithi111 ceph-mon[30255]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:33.501 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:34.901 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:34 smithi005 ceph-mon[26371]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:34.902 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:34 smithi005 ceph-mon[36961]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:34 smithi111 ceph-mon[30255]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:35.891 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:45:35.893 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":69,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T04:45:23.629378+0000","last_up_change":"2024-04-26T04:43:31.898273+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:44:07.074856+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:45:20.482057+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:45:21.051786+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6803","nonce":1943995172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6805","nonce":1943995172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6809","nonce":1943995172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6807","nonce":1943995172}]},"public_addr":"172.21.15.5:6803/1943995172","cluster_addr":"172.21.15.5:6805/1943995172","heartbeat_back_addr":"172.21.15.5:6809/1943995172","heartbeat_front_addr":"172.21.15.5:6807/1943995172","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6811","nonce":2192671947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6813","nonce":2192671947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6817","nonce":2192671947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6815","nonce":2192671947}]},"public_addr":"172.21.15.5:6811/2192671947","cluster_addr":"172.21.15.5:6813/2192671947","heartbeat_back_addr":"172.21.15.5:6817/2192671947","heartbeat_front_addr":"172.21.15.5:6815/2192671947","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6819","nonce":1951395348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6821","nonce":1951395348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6825","nonce":1951395348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6823","nonce":1951395348}]},"public_addr":"172.21.15.5:6819/1951395348","cluster_addr":"172.21.15.5:6821/1951395348","heartbeat_back_addr":"172.21.15.5:6825/1951395348","heartbeat_front_addr":"172.21.15.5:6823/1951395348","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6827","nonce":1448676980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6829","nonce":1448676980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6833","nonce":1448676980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6831","nonce":1448676980}]},"public_addr":"172.21.15.5:6827/1448676980","cluster_addr":"172.21.15.5:6829/1448676980","heartbeat_back_addr":"172.21.15.5:6833/1448676980","heartbeat_front_addr":"172.21.15.5:6831/1448676980","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/1382772406":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1663053230":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/3920149860":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3928321375":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:45:36.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:36] "GET /metrics HTTP/1.1" 200 208200 "" "Prometheus/2.43.0" 2024-04-26T04:45:36.615 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.615 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.615 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.615 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.615 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: Deploying daemon mds.foo.smithi111.furhfd on smithi111 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:boot 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: daemon mds.foo.smithi005.zdnlsb assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:45:36.616 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: fsmap foo:0 1 up:standby 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:creating} 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: daemon mds.foo.smithi005.zdnlsb is now active in filesystem foo as rank 0 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/968078257' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.617 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: Deploying daemon mds.foo.smithi111.furhfd on smithi111 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:boot 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: daemon mds.foo.smithi005.zdnlsb assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:45:36.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:45:36.619 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: fsmap foo:0 1 up:standby 2024-04-26T04:45:36.619 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:45:36.619 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:creating} 2024-04-26T04:45:36.619 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: daemon mds.foo.smithi005.zdnlsb is now active in filesystem foo as rank 0 2024-04-26T04:45:36.619 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:36 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/968078257' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:45:36.644 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:45:36.645 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:45:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: Deploying daemon mds.foo.smithi111.furhfd on smithi111 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:boot 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: daemon mds.foo.smithi005.zdnlsb assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: fsmap foo:0 1 up:standby 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:creating} 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: daemon mds.foo.smithi005.zdnlsb is now active in filesystem foo as rank 0 2024-04-26T04:45:36.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:36 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/968078257' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:45:36.914 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:37.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:37 smithi005 ceph-mon[26371]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:active 2024-04-26T04:45:37.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:37 smithi005 ceph-mon[26371]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 2024-04-26T04:45:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:37 smithi005 ceph-mon[36961]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:active 2024-04-26T04:45:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:37 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 2024-04-26T04:45:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:37 smithi111 ceph-mon[30255]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:active 2024-04-26T04:45:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:37 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 2024-04-26T04:45:38.867 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:45:38.867 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":69,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T04:45:23.629378+0000","last_up_change":"2024-04-26T04:43:31.898273+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:44:07.074856+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:45:20.482057+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:45:21.051786+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6803","nonce":1943995172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6805","nonce":1943995172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6809","nonce":1943995172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":1943995172},{"type":"v1","addr":"172.21.15.5:6807","nonce":1943995172}]},"public_addr":"172.21.15.5:6803/1943995172","cluster_addr":"172.21.15.5:6805/1943995172","heartbeat_back_addr":"172.21.15.5:6809/1943995172","heartbeat_front_addr":"172.21.15.5:6807/1943995172","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6811","nonce":2192671947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6813","nonce":2192671947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6817","nonce":2192671947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":2192671947},{"type":"v1","addr":"172.21.15.5:6815","nonce":2192671947}]},"public_addr":"172.21.15.5:6811/2192671947","cluster_addr":"172.21.15.5:6813/2192671947","heartbeat_back_addr":"172.21.15.5:6817/2192671947","heartbeat_front_addr":"172.21.15.5:6815/2192671947","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6819","nonce":1951395348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6821","nonce":1951395348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6825","nonce":1951395348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1951395348},{"type":"v1","addr":"172.21.15.5:6823","nonce":1951395348}]},"public_addr":"172.21.15.5:6819/1951395348","cluster_addr":"172.21.15.5:6821/1951395348","heartbeat_back_addr":"172.21.15.5:6825/1951395348","heartbeat_front_addr":"172.21.15.5:6823/1951395348","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6827","nonce":1448676980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6829","nonce":1448676980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6833","nonce":1448676980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":1448676980},{"type":"v1","addr":"172.21.15.5:6831","nonce":1448676980}]},"public_addr":"172.21.15.5:6827/1448676980","cluster_addr":"172.21.15.5:6829/1448676980","heartbeat_back_addr":"172.21.15.5:6833/1448676980","heartbeat_front_addr":"172.21.15.5:6831/1448676980","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/1382772406":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1663053230":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/3920149860":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3928321375":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:45:38.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:38.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:38 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:38 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[36961]: mds.? [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] up:boot 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3002842745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[36961]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:39.792 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[26371]: mds.? [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] up:boot 2024-04-26T04:45:39.793 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[26371]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:39.793 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:45:39.793 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[26371]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:39.793 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3002842745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:45:39.793 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:39 smithi005 ceph-mon[26371]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:39 smithi111 ceph-mon[30255]: mds.? [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] up:boot 2024-04-26T04:45:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:39 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:39 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:45:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:39 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:39 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3002842745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:45:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:39 smithi111 ceph-mon[30255]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:40.235 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:45:40.236 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:45:40.236 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:45:40.237 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:45:40.237 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:45:40.237 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:45:40.238 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:45:40.238 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:45:40.904 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.910 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.911 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.911 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.911 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.913 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.919 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:40.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:40 smithi005 ceph-mon[36961]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:active 2024-04-26T04:45:40.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:40 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:40.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:40 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:40 smithi005 ceph-mon[26371]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:active 2024-04-26T04:45:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:40 smithi005 ceph-mon[26371]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:40 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:40 smithi111 ceph-mon[30255]: mds.? [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] up:active 2024-04-26T04:45:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:40 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:40 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:41.048 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:41 smithi005 ceph-mon[36961]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:41 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:41 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:41 smithi005 ceph-mon[26371]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:41 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:41 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:41 smithi111 ceph-mon[30255]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:41 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:41 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:42.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:45:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:45:43.708 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:43 smithi005 ceph-mon[26371]: mds.? [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] up:standby 2024-04-26T04:45:43.708 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:43 smithi005 ceph-mon[26371]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:43.708 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:43 smithi005 ceph-mon[26371]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:43 smithi005 ceph-mon[36961]: mds.? [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] up:standby 2024-04-26T04:45:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:43 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:43 smithi005 ceph-mon[36961]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:43 smithi111 ceph-mon[30255]: mds.? [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] up:standby 2024-04-26T04:45:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:43 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:45:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:43 smithi111 ceph-mon[30255]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:46.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:46 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:46] "GET /metrics HTTP/1.1" 200 208200 "" "Prometheus/2.43.0" 2024-04-26T04:45:46.840 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:46 smithi005 ceph-mon[26371]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:46.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:46 smithi005 ceph-mon[36961]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:46 smithi111 ceph-mon[30255]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:48.656 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:48 smithi005 ceph-mon[26371]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:48.656 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:48 smithi005 ceph-mon[36961]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:48.683 INFO:teuthology.orchestra.run.smithi005.stdout:201863462946 2024-04-26T04:45:48.684 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:45:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:48 smithi111 ceph-mon[30255]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:45:50.033 INFO:teuthology.orchestra.run.smithi005.stdout:34359738433 2024-04-26T04:45:50.033 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:45:50.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:50 smithi005 ceph-mon[36961]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:50.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:50 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:50.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:50 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:50.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:50 smithi005 ceph-mon[26371]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:50.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:50 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:50.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:50 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:50 smithi111 ceph-mon[30255]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:50 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:50 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:52.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:45:51 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:45:52.740 INFO:teuthology.orchestra.run.smithi005.stdout:236223201310 2024-04-26T04:45:52.741 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:45:52.851 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:52 smithi005 ceph-mon[26371]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:52.851 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:52 smithi005 ceph-mon[36961]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:52 smithi111 ceph-mon[30255]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:54.511 INFO:teuthology.orchestra.run.smithi005.stdout:167503724585 2024-04-26T04:45:54.511 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:45:54.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:54 smithi005 ceph-mon[36961]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:54.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:54 smithi005 ceph-mon[26371]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:54 smithi111 ceph-mon[30255]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:45:56.311 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:45:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:45:56] "GET /metrics HTTP/1.1" 200 239218 "" "Prometheus/2.43.0" 2024-04-26T04:45:56.723 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:56 smithi005 ceph-mon[36961]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:56.724 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:56 smithi005 ceph-mon[26371]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:56 smithi111 ceph-mon[30255]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:57.164 INFO:teuthology.orchestra.run.smithi005.stdout:133143986220 2024-04-26T04:45:57.164 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:45:57.891 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[26371]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:45:57 smithi005 ceph-mon[36961]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:45:57 smithi111 ceph-mon[30255]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:58.469 INFO:teuthology.orchestra.run.smithi005.stdout:55834574908 2024-04-26T04:45:58.469 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:45:59.053 INFO:teuthology.orchestra.run.smithi005.stdout:111669149746 2024-04-26T04:45:59.053 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:46:00.030 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:00.042 INFO:teuthology.orchestra.run.smithi005.stdout:77309411385 2024-04-26T04:46:00.043 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:46:00.855 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:00 smithi005 ceph-mon[36961]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:00.855 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:00 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:00.855 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:00 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:00.856 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:00 smithi005 ceph-mon[26371]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:00.856 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:00 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:00.856 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:00 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:00 smithi111 ceph-mon[30255]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:00 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:00 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:01.199 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:01.205 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:01.205 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:01.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:46:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:46:02.100 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:02.702 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:02 smithi005 ceph-mon[26371]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:02.702 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:02 smithi005 ceph-mon[36961]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:02 smithi111 ceph-mon[30255]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:04.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:04 smithi005 ceph-mon[26371]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:04.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:04 smithi005 ceph-mon[36961]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:04 smithi111 ceph-mon[30255]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:05.662 INFO:teuthology.orchestra.run.smithi005.stdout:34359738436 2024-04-26T04:46:05.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:46:05.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:05 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:05 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:46:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:05 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-26T04:46:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:05 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:46:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:05 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:06.340 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:06] "GET /metrics HTTP/1.1" 200 239183 "" "Prometheus/2.43.0" 2024-04-26T04:46:06.375 INFO:teuthology.orchestra.run.smithi005.stdout:167503724587 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3119612825' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3095224237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:46:06.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-26T04:46:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-26T04:46:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-26T04:46:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:46:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3119612825' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:46:06.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:06 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3095224237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:46:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-26T04:46:07.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-26T04:46:07.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-26T04:46:07.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:46:07.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3119612825' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:46:07.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:06 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3095224237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:46:07.675 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:07 smithi005 ceph-mon[36961]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:46:07.675 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:07 smithi005 ceph-mon[26371]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:46:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:07 smithi111 ceph-mon[30255]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:46:08.102 INFO:teuthology.orchestra.run.smithi005.stdout:133143986223 2024-04-26T04:46:08.788 INFO:teuthology.orchestra.run.smithi005.stdout:201863462951 2024-04-26T04:46:08.788 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:08 smithi005 ceph-mon[36961]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:08.789 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:08 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2915195373' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:46:08.789 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:08 smithi005 ceph-mon[26371]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:08.789 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:08 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2915195373' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:46:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:08 smithi111 ceph-mon[30255]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:08 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2915195373' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:46:09.078 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986220 got 133143986223 for osd.4 2024-04-26T04:46:09.078 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:09.587 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738436 for osd.0 2024-04-26T04:46:09.587 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:09.852 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:09.853 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:09.906 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:09 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2505365161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:46:09.906 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:09 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2505365161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:46:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:09 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2505365161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:46:10.865 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:10 smithi005 ceph-mon[26371]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:10.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:10 smithi005 ceph-mon[36961]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:10.965 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:10 smithi111 ceph-mon[30255]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:11.405 INFO:teuthology.orchestra.run.smithi005.stdout:236223201314 2024-04-26T04:46:11.669 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:11 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3691135948' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:46:11.670 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:11 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3691135948' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:46:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:11 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3691135948' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:46:12.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:46:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:46:12.666 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724585 got 167503724587 for osd.5 2024-04-26T04:46:12.667 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:12.852 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:12 smithi005 ceph-mon[26371]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:12.852 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:12 smithi005 ceph-mon[36961]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:12 smithi111 ceph-mon[30255]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:13.060 INFO:teuthology.orchestra.run.smithi005.stdout:111669149749 2024-04-26T04:46:13.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:13 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3807611515' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:46:13.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:13 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3807611515' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:46:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:13 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3807611515' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:46:14.360 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201310 got 236223201314 for osd.7 2024-04-26T04:46:14.360 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:14.662 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462946 got 201863462951 for osd.6 2024-04-26T04:46:14.662 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:14.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:14 smithi005 ceph-mon[26371]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:14.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:14 smithi005 ceph-mon[36961]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:14 smithi111 ceph-mon[30255]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:15.450 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149749 for osd.3 2024-04-26T04:46:15.450 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:16.309 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:16] "GET /metrics HTTP/1.1" 200 239183 "" "Prometheus/2.43.0" 2024-04-26T04:46:16.455 INFO:teuthology.orchestra.run.smithi005.stdout:55834574912 2024-04-26T04:46:16.859 INFO:teuthology.orchestra.run.smithi005.stdout:77309411388 2024-04-26T04:46:16.859 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:16 smithi005 ceph-mon[26371]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:16.859 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:16 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2226303989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:46:16.860 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:16 smithi005 ceph-mon[36961]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:16.860 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:16 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2226303989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:46:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:16 smithi111 ceph-mon[30255]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:16 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2226303989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:46:17.411 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574908 got 55834574912 for osd.1 2024-04-26T04:46:17.411 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:17.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:17 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1715714246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:46:17.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:17 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1715714246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:46:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:17 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1715714246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:46:18.034 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411385 got 77309411388 for osd.2 2024-04-26T04:46:18.035 DEBUG:teuthology.parallel:result is None 2024-04-26T04:46:18.035 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:46:18.035 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:46:18.596 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:18 smithi005 ceph-mon[26371]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:18.731 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:18.861 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:18 smithi005 ceph-mon[36961]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:18 smithi111 ceph-mon[30255]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.854 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:20 smithi005 ceph-mon[36961]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.854 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:20 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:20.855 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:20 smithi005 ceph-mon[26371]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.855 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:20 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:20 smithi111 ceph-mon[30255]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:20 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:21.307 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:46:21.308 INFO:teuthology.orchestra.run.smithi005.stderr:dumped all 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:21.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:22.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:46:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:46:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:22.027 INFO:teuthology.orchestra.run.smithi005.stdout:{"pg_ready":true,"pg_map":{"version":81,"stamp":"2024-04-26T04:46:19.498219+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386344,"kb_used_data":7476,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574872,"statfs":{"total":767960285184,"available":765516668928,"internally_reserved":0,"allocated":7655424,"data_stored":4308280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002337"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.915863+0000","last_change":"2024-04-26T04:45:22.644600+0000","last_active":"2024-04-26T04:46:05.915863+0000","last_peered":"2024-04-26T04:46:05.915863+0000","last_clean":"2024-04-26T04:46:05.915863+0000","last_became_active":"2024-04-26T04:45:22.644313+0000","last_became_peered":"2024-04-26T04:45:22.644313+0000","last_unstale":"2024-04-26T04:46:05.915863+0000","last_undegraded":"2024-04-26T04:46:05.915863+0000","last_fullsized":"2024-04-26T04:46:05.915863+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:46:58.521750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559643+0000","last_change":"2024-04-26T04:45:21.641941+0000","last_active":"2024-04-26T04:46:05.559643+0000","last_peered":"2024-04-26T04:46:05.559643+0000","last_clean":"2024-04-26T04:46:05.559643+0000","last_became_active":"2024-04-26T04:45:21.639163+0000","last_became_peered":"2024-04-26T04:45:21.639163+0000","last_unstale":"2024-04-26T04:46:05.559643+0000","last_undegraded":"2024-04-26T04:46:05.559643+0000","last_fullsized":"2024-04-26T04:46:05.559643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:15:16.779001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562946+0000","last_change":"2024-04-26T04:45:22.651667+0000","last_active":"2024-04-26T04:46:05.562946+0000","last_peered":"2024-04-26T04:46:05.562946+0000","last_clean":"2024-04-26T04:46:05.562946+0000","last_became_active":"2024-04-26T04:45:22.651398+0000","last_became_peered":"2024-04-26T04:45:22.651398+0000","last_unstale":"2024-04-26T04:46:05.562946+0000","last_undegraded":"2024-04-26T04:46:05.562946+0000","last_fullsized":"2024-04-26T04:46:05.562946+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:31:57.490271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563741+0000","last_change":"2024-04-26T04:45:21.637069+0000","last_active":"2024-04-26T04:46:05.563741+0000","last_peered":"2024-04-26T04:46:05.563741+0000","last_clean":"2024-04-26T04:46:05.563741+0000","last_became_active":"2024-04-26T04:45:21.636722+0000","last_became_peered":"2024-04-26T04:45:21.636722+0000","last_unstale":"2024-04-26T04:46:05.563741+0000","last_undegraded":"2024-04-26T04:46:05.563741+0000","last_fullsized":"2024-04-26T04:46:05.563741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:14:27.222222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916090+0000","last_change":"2024-04-26T04:45:22.648085+0000","last_active":"2024-04-26T04:46:05.916090+0000","last_peered":"2024-04-26T04:46:05.916090+0000","last_clean":"2024-04-26T04:46:05.916090+0000","last_became_active":"2024-04-26T04:45:22.647782+0000","last_became_peered":"2024-04-26T04:45:22.647782+0000","last_unstale":"2024-04-26T04:46:05.916090+0000","last_undegraded":"2024-04-26T04:46:05.916090+0000","last_fullsized":"2024-04-26T04:46:05.916090+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:37:12.680619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560575+0000","last_change":"2024-04-26T04:45:21.634660+0000","last_active":"2024-04-26T04:46:05.560575+0000","last_peered":"2024-04-26T04:46:05.560575+0000","last_clean":"2024-04-26T04:46:05.560575+0000","last_became_active":"2024-04-26T04:45:21.634404+0000","last_became_peered":"2024-04-26T04:45:21.634404+0000","last_unstale":"2024-04-26T04:46:05.560575+0000","last_undegraded":"2024-04-26T04:46:05.560575+0000","last_fullsized":"2024-04-26T04:46:05.560575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:28:20.390001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916482+0000","last_change":"2024-04-26T04:45:22.648270+0000","last_active":"2024-04-26T04:46:05.916482+0000","last_peered":"2024-04-26T04:46:05.916482+0000","last_clean":"2024-04-26T04:46:05.916482+0000","last_became_active":"2024-04-26T04:45:22.647940+0000","last_became_peered":"2024-04-26T04:45:22.647940+0000","last_unstale":"2024-04-26T04:46:05.916482+0000","last_undegraded":"2024-04-26T04:46:05.916482+0000","last_fullsized":"2024-04-26T04:46:05.916482+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:14:16.459554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560487+0000","last_change":"2024-04-26T04:45:21.641206+0000","last_active":"2024-04-26T04:46:05.560487+0000","last_peered":"2024-04-26T04:46:05.560487+0000","last_clean":"2024-04-26T04:46:05.560487+0000","last_became_active":"2024-04-26T04:45:21.640887+0000","last_became_peered":"2024-04-26T04:45:21.640887+0000","last_unstale":"2024-04-26T04:46:05.560487+0000","last_undegraded":"2024-04-26T04:46:05.560487+0000","last_fullsized":"2024-04-26T04:46:05.560487+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:04:06.041378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:46:06.882564+0000","last_change":"2024-04-26T04:46:06.882564+0000","last_active":"2024-04-26T04:46:06.882564+0000","last_peered":"2024-04-26T04:46:06.882564+0000","last_clean":"2024-04-26T04:46:06.882564+0000","last_became_active":"2024-04-26T04:46:06.882204+0000","last_became_peered":"2024-04-26T04:46:06.882204+0000","last_unstale":"2024-04-26T04:46:06.882564+0000","last_undegraded":"2024-04-26T04:46:06.882564+0000","last_fullsized":"2024-04-26T04:46:06.882564+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:06:15.186341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559264+0000","last_change":"2024-04-26T04:45:21.631557+0000","last_active":"2024-04-26T04:46:05.559264+0000","last_peered":"2024-04-26T04:46:05.559264+0000","last_clean":"2024-04-26T04:46:05.559264+0000","last_became_active":"2024-04-26T04:45:21.630609+0000","last_became_peered":"2024-04-26T04:45:21.630609+0000","last_unstale":"2024-04-26T04:46:05.559264+0000","last_undegraded":"2024-04-26T04:46:05.559264+0000","last_fullsized":"2024-04-26T04:46:05.559264+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:14:38.394486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916088+0000","last_change":"2024-04-26T04:45:22.644110+0000","last_active":"2024-04-26T04:46:05.916088+0000","last_peered":"2024-04-26T04:46:05.916088+0000","last_clean":"2024-04-26T04:46:05.916088+0000","last_became_active":"2024-04-26T04:45:22.643811+0000","last_became_peered":"2024-04-26T04:45:22.643811+0000","last_unstale":"2024-04-26T04:46:05.916088+0000","last_undegraded":"2024-04-26T04:46:05.916088+0000","last_fullsized":"2024-04-26T04:46:05.916088+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:20:05.112184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559335+0000","last_change":"2024-04-26T04:45:21.625983+0000","last_active":"2024-04-26T04:46:05.559335+0000","last_peered":"2024-04-26T04:46:05.559335+0000","last_clean":"2024-04-26T04:46:05.559335+0000","last_became_active":"2024-04-26T04:45:21.625586+0000","last_became_peered":"2024-04-26T04:45:21.625586+0000","last_unstale":"2024-04-26T04:46:05.559335+0000","last_undegraded":"2024-04-26T04:46:05.559335+0000","last_fullsized":"2024-04-26T04:46:05.559335+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:46:48.199296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563104+0000","last_change":"2024-04-26T04:44:08.479180+0000","last_active":"2024-04-26T04:46:05.563104+0000","last_peered":"2024-04-26T04:46:05.563104+0000","last_clean":"2024-04-26T04:46:05.563104+0000","last_became_active":"2024-04-26T04:44:08.478822+0000","last_became_peered":"2024-04-26T04:44:08.478822+0000","last_unstale":"2024-04-26T04:46:05.563104+0000","last_undegraded":"2024-04-26T04:46:05.563104+0000","last_fullsized":"2024-04-26T04:46:05.563104+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:08:10.312422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562442+0000","last_change":"2024-04-26T04:45:22.648080+0000","last_active":"2024-04-26T04:46:05.562442+0000","last_peered":"2024-04-26T04:46:05.562442+0000","last_clean":"2024-04-26T04:46:05.562442+0000","last_became_active":"2024-04-26T04:45:22.647748+0000","last_became_peered":"2024-04-26T04:45:22.647748+0000","last_unstale":"2024-04-26T04:46:05.562442+0000","last_undegraded":"2024-04-26T04:46:05.562442+0000","last_fullsized":"2024-04-26T04:46:05.562442+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:05:47.555778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560719+0000","last_change":"2024-04-26T04:44:08.965639+0000","last_active":"2024-04-26T04:46:05.560719+0000","last_peered":"2024-04-26T04:46:05.560719+0000","last_clean":"2024-04-26T04:46:05.560719+0000","last_became_active":"2024-04-26T04:44:08.965296+0000","last_became_peered":"2024-04-26T04:44:08.965296+0000","last_unstale":"2024-04-26T04:46:05.560719+0000","last_undegraded":"2024-04-26T04:46:05.560719+0000","last_fullsized":"2024-04-26T04:46:05.560719+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:17:21.026489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563799+0000","last_change":"2024-04-26T04:45:21.645833+0000","last_active":"2024-04-26T04:46:05.563799+0000","last_peered":"2024-04-26T04:46:05.563799+0000","last_clean":"2024-04-26T04:46:05.563799+0000","last_became_active":"2024-04-26T04:45:21.645542+0000","last_became_peered":"2024-04-26T04:45:21.645542+0000","last_unstale":"2024-04-26T04:46:05.563799+0000","last_undegraded":"2024-04-26T04:46:05.563799+0000","last_fullsized":"2024-04-26T04:46:05.563799+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:44:25.617808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559183+0000","last_change":"2024-04-26T04:45:22.642805+0000","last_active":"2024-04-26T04:46:05.559183+0000","last_peered":"2024-04-26T04:46:05.559183+0000","last_clean":"2024-04-26T04:46:05.559183+0000","last_became_active":"2024-04-26T04:45:22.642314+0000","last_became_peered":"2024-04-26T04:45:22.642314+0000","last_unstale":"2024-04-26T04:46:05.559183+0000","last_undegraded":"2024-04-26T04:46:05.559183+0000","last_fullsized":"2024-04-26T04:46:05.559183+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:10:38.308347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.561073+0000","last_change":"2024-04-26T04:45:21.636871+0000","last_active":"2024-04-26T04:46:05.561073+0000","last_peered":"2024-04-26T04:46:05.561073+0000","last_clean":"2024-04-26T04:46:05.561073+0000","last_became_active":"2024-04-26T04:45:21.636389+0000","last_became_peered":"2024-04-26T04:45:21.636389+0000","last_unstale":"2024-04-26T04:46:05.561073+0000","last_undegraded":"2024-04-26T04:46:05.561073+0000","last_fullsized":"2024-04-26T04:46:05.561073+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:30:22.877766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565470+0000","last_change":"2024-04-26T04:44:08.965486+0000","last_active":"2024-04-26T04:46:05.565470+0000","last_peered":"2024-04-26T04:46:05.565470+0000","last_clean":"2024-04-26T04:46:05.565470+0000","last_became_active":"2024-04-26T04:44:08.965051+0000","last_became_peered":"2024-04-26T04:44:08.965051+0000","last_unstale":"2024-04-26T04:46:05.565470+0000","last_undegraded":"2024-04-26T04:46:05.565470+0000","last_fullsized":"2024-04-26T04:46:05.565470+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:47:18.277374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559353+0000","last_change":"2024-04-26T04:45:22.641594+0000","last_active":"2024-04-26T04:46:05.559353+0000","last_peered":"2024-04-26T04:46:05.559353+0000","last_clean":"2024-04-26T04:46:05.559353+0000","last_became_active":"2024-04-26T04:45:22.641288+0000","last_became_peered":"2024-04-26T04:45:22.641288+0000","last_unstale":"2024-04-26T04:46:05.559353+0000","last_undegraded":"2024-04-26T04:46:05.559353+0000","last_fullsized":"2024-04-26T04:46:05.559353+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:38:39.016111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.558932+0000","last_change":"2024-04-26T04:45:21.639327+0000","last_active":"2024-04-26T04:46:05.558932+0000","last_peered":"2024-04-26T04:46:05.558932+0000","last_clean":"2024-04-26T04:46:05.558932+0000","last_became_active":"2024-04-26T04:45:21.638898+0000","last_became_peered":"2024-04-26T04:45:21.638898+0000","last_unstale":"2024-04-26T04:46:05.558932+0000","last_undegraded":"2024-04-26T04:46:05.558932+0000","last_fullsized":"2024-04-26T04:46:05.558932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:15:11.955760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565195+0000","last_change":"2024-04-26T04:44:08.481145+0000","last_active":"2024-04-26T04:46:05.565195+0000","last_peered":"2024-04-26T04:46:05.565195+0000","last_clean":"2024-04-26T04:46:05.565195+0000","last_became_active":"2024-04-26T04:44:08.480765+0000","last_became_peered":"2024-04-26T04:44:08.480765+0000","last_unstale":"2024-04-26T04:46:05.565195+0000","last_undegraded":"2024-04-26T04:46:05.565195+0000","last_fullsized":"2024-04-26T04:46:05.565195+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:48:08.184965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563055+0000","last_change":"2024-04-26T04:45:22.645366+0000","last_active":"2024-04-26T04:46:05.563055+0000","last_peered":"2024-04-26T04:46:05.563055+0000","last_clean":"2024-04-26T04:46:05.563055+0000","last_became_active":"2024-04-26T04:45:22.645130+0000","last_became_peered":"2024-04-26T04:45:22.645130+0000","last_unstale":"2024-04-26T04:46:05.563055+0000","last_undegraded":"2024-04-26T04:46:05.563055+0000","last_fullsized":"2024-04-26T04:46:05.563055+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:52:52.129108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562606+0000","last_change":"2024-04-26T04:45:21.637077+0000","last_active":"2024-04-26T04:46:05.562606+0000","last_peered":"2024-04-26T04:46:05.562606+0000","last_clean":"2024-04-26T04:46:05.562606+0000","last_became_active":"2024-04-26T04:45:21.636047+0000","last_became_peered":"2024-04-26T04:45:21.636047+0000","last_unstale":"2024-04-26T04:46:05.562606+0000","last_undegraded":"2024-04-26T04:46:05.562606+0000","last_fullsized":"2024-04-26T04:46:05.562606+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:53:59.428390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":278,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565171+0000","last_change":"2024-04-26T04:43:35.916221+0000","last_active":"2024-04-26T04:46:05.565171+0000","last_peered":"2024-04-26T04:46:05.565171+0000","last_clean":"2024-04-26T04:46:05.565171+0000","last_became_active":"2024-04-26T04:43:35.915728+0000","last_became_peered":"2024-04-26T04:43:35.915728+0000","last_unstale":"2024-04-26T04:46:05.565171+0000","last_undegraded":"2024-04-26T04:46:05.565171+0000","last_fullsized":"2024-04-26T04:46:05.565171+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_clean_scrub_stamp":"2024-04-26T04:41:33.492763+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:47:26.279649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560529+0000","last_change":"2024-04-26T04:44:08.479414+0000","last_active":"2024-04-26T04:46:05.560529+0000","last_peered":"2024-04-26T04:46:05.560529+0000","last_clean":"2024-04-26T04:46:05.560529+0000","last_became_active":"2024-04-26T04:44:08.478584+0000","last_became_peered":"2024-04-26T04:44:08.478584+0000","last_unstale":"2024-04-26T04:46:05.560529+0000","last_undegraded":"2024-04-26T04:46:05.560529+0000","last_fullsized":"2024-04-26T04:46:05.560529+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:02:29.012874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560401+0000","last_change":"2024-04-26T04:45:22.640395+0000","last_active":"2024-04-26T04:46:05.560401+0000","last_peered":"2024-04-26T04:46:05.560401+0000","last_clean":"2024-04-26T04:46:05.560401+0000","last_became_active":"2024-04-26T04:45:22.640113+0000","last_became_peered":"2024-04-26T04:45:22.640113+0000","last_unstale":"2024-04-26T04:46:05.560401+0000","last_undegraded":"2024-04-26T04:46:05.560401+0000","last_fullsized":"2024-04-26T04:46:05.560401+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:58:17.399345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559817+0000","last_change":"2024-04-26T04:45:21.628785+0000","last_active":"2024-04-26T04:46:05.559817+0000","last_peered":"2024-04-26T04:46:05.559817+0000","last_clean":"2024-04-26T04:46:05.559817+0000","last_became_active":"2024-04-26T04:45:21.628460+0000","last_became_peered":"2024-04-26T04:45:21.628460+0000","last_unstale":"2024-04-26T04:46:05.559817+0000","last_undegraded":"2024-04-26T04:46:05.559817+0000","last_fullsized":"2024-04-26T04:46:05.559817+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:05:42.764810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562756+0000","last_change":"2024-04-26T04:44:09.231105+0000","last_active":"2024-04-26T04:46:05.562756+0000","last_peered":"2024-04-26T04:46:05.562756+0000","last_clean":"2024-04-26T04:46:05.562756+0000","last_became_active":"2024-04-26T04:44:09.230737+0000","last_became_peered":"2024-04-26T04:44:09.230737+0000","last_unstale":"2024-04-26T04:46:05.562756+0000","last_undegraded":"2024-04-26T04:46:05.562756+0000","last_fullsized":"2024-04-26T04:46:05.562756+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:25:04.219657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.561030+0000","last_change":"2024-04-26T04:45:22.644736+0000","last_active":"2024-04-26T04:46:05.561030+0000","last_peered":"2024-04-26T04:46:05.561030+0000","last_clean":"2024-04-26T04:46:05.561030+0000","last_became_active":"2024-04-26T04:45:22.644457+0000","last_became_peered":"2024-04-26T04:45:22.644457+0000","last_unstale":"2024-04-26T04:46:05.561030+0000","last_undegraded":"2024-04-26T04:46:05.561030+0000","last_fullsized":"2024-04-26T04:46:05.561030+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:42:51.554720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916199+0000","last_change":"2024-04-26T04:45:21.636903+0000","last_active":"2024-04-26T04:46:05.916199+0000","last_peered":"2024-04-26T04:46:05.916199+0000","last_clean":"2024-04-26T04:46:05.916199+0000","last_became_active":"2024-04-26T04:45:21.635678+0000","last_became_peered":"2024-04-26T04:45:21.635678+0000","last_unstale":"2024-04-26T04:46:05.916199+0000","last_undegraded":"2024-04-26T04:46:05.916199+0000","last_fullsized":"2024-04-26T04:46:05.916199+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:17:03.267381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560419+0000","last_change":"2024-04-26T04:44:08.477772+0000","last_active":"2024-04-26T04:46:05.560419+0000","last_peered":"2024-04-26T04:46:05.560419+0000","last_clean":"2024-04-26T04:46:05.560419+0000","last_became_active":"2024-04-26T04:44:08.477431+0000","last_became_peered":"2024-04-26T04:44:08.477431+0000","last_unstale":"2024-04-26T04:46:05.560419+0000","last_undegraded":"2024-04-26T04:46:05.560419+0000","last_fullsized":"2024-04-26T04:46:05.560419+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:50:07.493196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560354+0000","last_change":"2024-04-26T04:45:22.642296+0000","last_active":"2024-04-26T04:46:05.560354+0000","last_peered":"2024-04-26T04:46:05.560354+0000","last_clean":"2024-04-26T04:46:05.560354+0000","last_became_active":"2024-04-26T04:45:22.642060+0000","last_became_peered":"2024-04-26T04:45:22.642060+0000","last_unstale":"2024-04-26T04:46:05.560354+0000","last_undegraded":"2024-04-26T04:46:05.560354+0000","last_fullsized":"2024-04-26T04:46:05.560354+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:19:16.633439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560510+0000","last_change":"2024-04-26T04:45:21.636592+0000","last_active":"2024-04-26T04:46:05.560510+0000","last_peered":"2024-04-26T04:46:05.560510+0000","last_clean":"2024-04-26T04:46:05.560510+0000","last_became_active":"2024-04-26T04:45:21.636257+0000","last_became_peered":"2024-04-26T04:45:21.636257+0000","last_unstale":"2024-04-26T04:46:05.560510+0000","last_undegraded":"2024-04-26T04:46:05.560510+0000","last_fullsized":"2024-04-26T04:46:05.560510+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:28:18.062261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560935+0000","last_change":"2024-04-26T04:44:08.480906+0000","last_active":"2024-04-26T04:46:05.560935+0000","last_peered":"2024-04-26T04:46:05.560935+0000","last_clean":"2024-04-26T04:46:05.560935+0000","last_became_active":"2024-04-26T04:44:08.480613+0000","last_became_peered":"2024-04-26T04:44:08.480613+0000","last_unstale":"2024-04-26T04:46:05.560935+0000","last_undegraded":"2024-04-26T04:46:05.560935+0000","last_fullsized":"2024-04-26T04:46:05.560935+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:20:31.237343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560487+0000","last_change":"2024-04-26T04:45:22.643691+0000","last_active":"2024-04-26T04:46:05.560487+0000","last_peered":"2024-04-26T04:46:05.560487+0000","last_clean":"2024-04-26T04:46:05.560487+0000","last_became_active":"2024-04-26T04:45:22.643281+0000","last_became_peered":"2024-04-26T04:45:22.643281+0000","last_unstale":"2024-04-26T04:46:05.560487+0000","last_undegraded":"2024-04-26T04:46:05.560487+0000","last_fullsized":"2024-04-26T04:46:05.560487+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:34:41.476802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562456+0000","last_change":"2024-04-26T04:45:21.645718+0000","last_active":"2024-04-26T04:46:05.562456+0000","last_peered":"2024-04-26T04:46:05.562456+0000","last_clean":"2024-04-26T04:46:05.562456+0000","last_became_active":"2024-04-26T04:45:21.645368+0000","last_became_peered":"2024-04-26T04:45:21.645368+0000","last_unstale":"2024-04-26T04:46:05.562456+0000","last_undegraded":"2024-04-26T04:46:05.562456+0000","last_fullsized":"2024-04-26T04:46:05.562456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:41:26.200426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916354+0000","last_change":"2024-04-26T04:45:22.643766+0000","last_active":"2024-04-26T04:46:05.916354+0000","last_peered":"2024-04-26T04:46:05.916354+0000","last_clean":"2024-04-26T04:46:05.916354+0000","last_became_active":"2024-04-26T04:45:22.643486+0000","last_became_peered":"2024-04-26T04:45:22.643486+0000","last_unstale":"2024-04-26T04:46:05.916354+0000","last_undegraded":"2024-04-26T04:46:05.916354+0000","last_fullsized":"2024-04-26T04:46:05.916354+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:04:44.877539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916126+0000","last_change":"2024-04-26T04:45:21.635507+0000","last_active":"2024-04-26T04:46:05.916126+0000","last_peered":"2024-04-26T04:46:05.916126+0000","last_clean":"2024-04-26T04:46:05.916126+0000","last_became_active":"2024-04-26T04:45:21.634530+0000","last_became_peered":"2024-04-26T04:45:21.634530+0000","last_unstale":"2024-04-26T04:46:05.916126+0000","last_undegraded":"2024-04-26T04:46:05.916126+0000","last_fullsized":"2024-04-26T04:46:05.916126+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:36:40.790413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916519+0000","last_change":"2024-04-26T04:45:22.644087+0000","last_active":"2024-04-26T04:46:05.916519+0000","last_peered":"2024-04-26T04:46:05.916519+0000","last_clean":"2024-04-26T04:46:05.916519+0000","last_became_active":"2024-04-26T04:45:22.643763+0000","last_became_peered":"2024-04-26T04:45:22.643763+0000","last_unstale":"2024-04-26T04:46:05.916519+0000","last_undegraded":"2024-04-26T04:46:05.916519+0000","last_fullsized":"2024-04-26T04:46:05.916519+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:20:27.062806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562841+0000","last_change":"2024-04-26T04:45:21.636924+0000","last_active":"2024-04-26T04:46:05.562841+0000","last_peered":"2024-04-26T04:46:05.562841+0000","last_clean":"2024-04-26T04:46:05.562841+0000","last_became_active":"2024-04-26T04:45:21.636646+0000","last_became_peered":"2024-04-26T04:45:21.636646+0000","last_unstale":"2024-04-26T04:46:05.562841+0000","last_undegraded":"2024-04-26T04:46:05.562841+0000","last_fullsized":"2024-04-26T04:46:05.562841+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:45:22.147993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916221+0000","last_change":"2024-04-26T04:45:22.648565+0000","last_active":"2024-04-26T04:46:05.916221+0000","last_peered":"2024-04-26T04:46:05.916221+0000","last_clean":"2024-04-26T04:46:05.916221+0000","last_became_active":"2024-04-26T04:45:22.647969+0000","last_became_peered":"2024-04-26T04:45:22.647969+0000","last_unstale":"2024-04-26T04:46:05.916221+0000","last_undegraded":"2024-04-26T04:46:05.916221+0000","last_fullsized":"2024-04-26T04:46:05.916221+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:11.200395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563873+0000","last_change":"2024-04-26T04:45:21.645388+0000","last_active":"2024-04-26T04:46:05.563873+0000","last_peered":"2024-04-26T04:46:05.563873+0000","last_clean":"2024-04-26T04:46:05.563873+0000","last_became_active":"2024-04-26T04:45:21.645053+0000","last_became_peered":"2024-04-26T04:45:21.645053+0000","last_unstale":"2024-04-26T04:46:05.563873+0000","last_undegraded":"2024-04-26T04:46:05.563873+0000","last_fullsized":"2024-04-26T04:46:05.563873+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:38:37.713482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559109+0000","last_change":"2024-04-26T04:45:22.642590+0000","last_active":"2024-04-26T04:46:05.559109+0000","last_peered":"2024-04-26T04:46:05.559109+0000","last_clean":"2024-04-26T04:46:05.559109+0000","last_became_active":"2024-04-26T04:45:22.641884+0000","last_became_peered":"2024-04-26T04:45:22.641884+0000","last_unstale":"2024-04-26T04:46:05.559109+0000","last_undegraded":"2024-04-26T04:46:05.559109+0000","last_fullsized":"2024-04-26T04:46:05.559109+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:35:53.039625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560487+0000","last_change":"2024-04-26T04:45:21.631299+0000","last_active":"2024-04-26T04:46:05.560487+0000","last_peered":"2024-04-26T04:46:05.560487+0000","last_clean":"2024-04-26T04:46:05.560487+0000","last_became_active":"2024-04-26T04:45:21.630990+0000","last_became_peered":"2024-04-26T04:45:21.630990+0000","last_unstale":"2024-04-26T04:46:05.560487+0000","last_undegraded":"2024-04-26T04:46:05.560487+0000","last_fullsized":"2024-04-26T04:46:05.560487+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:36:19.517763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:46:06.568711+0000","last_change":"2024-04-26T04:46:06.568711+0000","last_active":"2024-04-26T04:46:06.568711+0000","last_peered":"2024-04-26T04:46:06.568711+0000","last_clean":"2024-04-26T04:46:06.568711+0000","last_became_active":"2024-04-26T04:46:06.568385+0000","last_became_peered":"2024-04-26T04:46:06.568385+0000","last_unstale":"2024-04-26T04:46:06.568711+0000","last_undegraded":"2024-04-26T04:46:06.568711+0000","last_fullsized":"2024-04-26T04:46:06.568711+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:25:21.170929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565572+0000","last_change":"2024-04-26T04:45:21.638699+0000","last_active":"2024-04-26T04:46:05.565572+0000","last_peered":"2024-04-26T04:46:05.565572+0000","last_clean":"2024-04-26T04:46:05.565572+0000","last_became_active":"2024-04-26T04:45:21.638390+0000","last_became_peered":"2024-04-26T04:45:21.638390+0000","last_unstale":"2024-04-26T04:46:05.565572+0000","last_undegraded":"2024-04-26T04:46:05.565572+0000","last_fullsized":"2024-04-26T04:46:05.565572+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:29:59.647984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.915900+0000","last_change":"2024-04-26T04:45:22.648384+0000","last_active":"2024-04-26T04:46:05.915900+0000","last_peered":"2024-04-26T04:46:05.915900+0000","last_clean":"2024-04-26T04:46:05.915900+0000","last_became_active":"2024-04-26T04:45:22.647989+0000","last_became_peered":"2024-04-26T04:45:22.647989+0000","last_unstale":"2024-04-26T04:46:05.915900+0000","last_undegraded":"2024-04-26T04:46:05.915900+0000","last_fullsized":"2024-04-26T04:46:05.915900+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:43:56.351981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565661+0000","last_change":"2024-04-26T04:45:21.643690+0000","last_active":"2024-04-26T04:46:05.565661+0000","last_peered":"2024-04-26T04:46:05.565661+0000","last_clean":"2024-04-26T04:46:05.565661+0000","last_became_active":"2024-04-26T04:45:21.643239+0000","last_became_peered":"2024-04-26T04:45:21.643239+0000","last_unstale":"2024-04-26T04:46:05.565661+0000","last_undegraded":"2024-04-26T04:46:05.565661+0000","last_fullsized":"2024-04-26T04:46:05.565661+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:47:09.366443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560512+0000","last_change":"2024-04-26T04:45:22.642459+0000","last_active":"2024-04-26T04:46:05.560512+0000","last_peered":"2024-04-26T04:46:05.560512+0000","last_clean":"2024-04-26T04:46:05.560512+0000","last_became_active":"2024-04-26T04:45:22.642177+0000","last_became_peered":"2024-04-26T04:45:22.642177+0000","last_unstale":"2024-04-26T04:46:05.560512+0000","last_undegraded":"2024-04-26T04:46:05.560512+0000","last_fullsized":"2024-04-26T04:46:05.560512+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:14.494066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565649+0000","last_change":"2024-04-26T04:45:21.634985+0000","last_active":"2024-04-26T04:46:05.565649+0000","last_peered":"2024-04-26T04:46:05.565649+0000","last_clean":"2024-04-26T04:46:05.565649+0000","last_became_active":"2024-04-26T04:45:21.634663+0000","last_became_peered":"2024-04-26T04:45:21.634663+0000","last_unstale":"2024-04-26T04:46:05.565649+0000","last_undegraded":"2024-04-26T04:46:05.565649+0000","last_fullsized":"2024-04-26T04:46:05.565649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:30:52.426028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563681+0000","last_change":"2024-04-26T04:45:22.647497+0000","last_active":"2024-04-26T04:46:05.563681+0000","last_peered":"2024-04-26T04:46:05.563681+0000","last_clean":"2024-04-26T04:46:05.563681+0000","last_became_active":"2024-04-26T04:45:22.647217+0000","last_became_peered":"2024-04-26T04:45:22.647217+0000","last_unstale":"2024-04-26T04:46:05.563681+0000","last_undegraded":"2024-04-26T04:46:05.563681+0000","last_fullsized":"2024-04-26T04:46:05.563681+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:54:55.108116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562726+0000","last_change":"2024-04-26T04:45:21.639205+0000","last_active":"2024-04-26T04:46:05.562726+0000","last_peered":"2024-04-26T04:46:05.562726+0000","last_clean":"2024-04-26T04:46:05.562726+0000","last_became_active":"2024-04-26T04:45:21.638924+0000","last_became_peered":"2024-04-26T04:45:21.638924+0000","last_unstale":"2024-04-26T04:46:05.562726+0000","last_undegraded":"2024-04-26T04:46:05.562726+0000","last_fullsized":"2024-04-26T04:46:05.562726+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:29:07.972721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559184+0000","last_change":"2024-04-26T04:45:22.650422+0000","last_active":"2024-04-26T04:46:05.559184+0000","last_peered":"2024-04-26T04:46:05.559184+0000","last_clean":"2024-04-26T04:46:05.559184+0000","last_became_active":"2024-04-26T04:45:22.650089+0000","last_became_peered":"2024-04-26T04:45:22.650089+0000","last_unstale":"2024-04-26T04:46:05.559184+0000","last_undegraded":"2024-04-26T04:46:05.559184+0000","last_fullsized":"2024-04-26T04:46:05.559184+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:38:42.148255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565122+0000","last_change":"2024-04-26T04:45:21.634430+0000","last_active":"2024-04-26T04:46:05.565122+0000","last_peered":"2024-04-26T04:46:05.565122+0000","last_clean":"2024-04-26T04:46:05.565122+0000","last_became_active":"2024-04-26T04:45:21.634149+0000","last_became_peered":"2024-04-26T04:45:21.634149+0000","last_unstale":"2024-04-26T04:46:05.565122+0000","last_undegraded":"2024-04-26T04:46:05.565122+0000","last_fullsized":"2024-04-26T04:46:05.565122+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:32.326287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560432+0000","last_change":"2024-04-26T04:45:22.642712+0000","last_active":"2024-04-26T04:46:05.560432+0000","last_peered":"2024-04-26T04:46:05.560432+0000","last_clean":"2024-04-26T04:46:05.560432+0000","last_became_active":"2024-04-26T04:45:22.641574+0000","last_became_peered":"2024-04-26T04:45:22.641574+0000","last_unstale":"2024-04-26T04:46:05.560432+0000","last_undegraded":"2024-04-26T04:46:05.560432+0000","last_fullsized":"2024-04-26T04:46:05.560432+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:26:03.339050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559083+0000","last_change":"2024-04-26T04:45:21.640861+0000","last_active":"2024-04-26T04:46:05.559083+0000","last_peered":"2024-04-26T04:46:05.559083+0000","last_clean":"2024-04-26T04:46:05.559083+0000","last_became_active":"2024-04-26T04:45:21.639828+0000","last_became_peered":"2024-04-26T04:45:21.639828+0000","last_unstale":"2024-04-26T04:46:05.559083+0000","last_undegraded":"2024-04-26T04:46:05.559083+0000","last_fullsized":"2024-04-26T04:46:05.559083+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:04:14.253122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:46:06.568107+0000","last_change":"2024-04-26T04:46:06.568107+0000","last_active":"2024-04-26T04:46:06.568107+0000","last_peered":"2024-04-26T04:46:06.568107+0000","last_clean":"2024-04-26T04:46:06.568107+0000","last_became_active":"2024-04-26T04:46:06.567748+0000","last_became_peered":"2024-04-26T04:46:06.567748+0000","last_unstale":"2024-04-26T04:46:06.568107+0000","last_undegraded":"2024-04-26T04:46:06.568107+0000","last_fullsized":"2024-04-26T04:46:06.568107+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:55:12.034909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565428+0000","last_change":"2024-04-26T04:45:21.643210+0000","last_active":"2024-04-26T04:46:05.565428+0000","last_peered":"2024-04-26T04:46:05.565428+0000","last_clean":"2024-04-26T04:46:05.565428+0000","last_became_active":"2024-04-26T04:45:21.642825+0000","last_became_peered":"2024-04-26T04:45:21.642825+0000","last_unstale":"2024-04-26T04:46:05.565428+0000","last_undegraded":"2024-04-26T04:46:05.565428+0000","last_fullsized":"2024-04-26T04:46:05.565428+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:17.053860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916344+0000","last_change":"2024-04-26T04:45:22.647005+0000","last_active":"2024-04-26T04:46:05.916344+0000","last_peered":"2024-04-26T04:46:05.916344+0000","last_clean":"2024-04-26T04:46:05.916344+0000","last_became_active":"2024-04-26T04:45:22.646753+0000","last_became_peered":"2024-04-26T04:45:22.646753+0000","last_unstale":"2024-04-26T04:46:05.916344+0000","last_undegraded":"2024-04-26T04:46:05.916344+0000","last_fullsized":"2024-04-26T04:46:05.916344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:22:11.999704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916285+0000","last_change":"2024-04-26T04:45:21.631617+0000","last_active":"2024-04-26T04:46:05.916285+0000","last_peered":"2024-04-26T04:46:05.916285+0000","last_clean":"2024-04-26T04:46:05.916285+0000","last_became_active":"2024-04-26T04:45:21.631218+0000","last_became_peered":"2024-04-26T04:45:21.631218+0000","last_unstale":"2024-04-26T04:46:05.916285+0000","last_undegraded":"2024-04-26T04:46:05.916285+0000","last_fullsized":"2024-04-26T04:46:05.916285+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:50:12.166307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560161+0000","last_change":"2024-04-26T04:45:22.644166+0000","last_active":"2024-04-26T04:46:05.560161+0000","last_peered":"2024-04-26T04:46:05.560161+0000","last_clean":"2024-04-26T04:46:05.560161+0000","last_became_active":"2024-04-26T04:45:22.643900+0000","last_became_peered":"2024-04-26T04:45:22.643900+0000","last_unstale":"2024-04-26T04:46:05.560161+0000","last_undegraded":"2024-04-26T04:46:05.560161+0000","last_fullsized":"2024-04-26T04:46:05.560161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:37.065136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565456+0000","last_change":"2024-04-26T04:45:21.641364+0000","last_active":"2024-04-26T04:46:05.565456+0000","last_peered":"2024-04-26T04:46:05.565456+0000","last_clean":"2024-04-26T04:46:05.565456+0000","last_became_active":"2024-04-26T04:45:21.640716+0000","last_became_peered":"2024-04-26T04:45:21.640716+0000","last_unstale":"2024-04-26T04:46:05.565456+0000","last_undegraded":"2024-04-26T04:46:05.565456+0000","last_fullsized":"2024-04-26T04:46:05.565456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:51:16.247487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563011+0000","last_change":"2024-04-26T04:45:22.646855+0000","last_active":"2024-04-26T04:46:05.563011+0000","last_peered":"2024-04-26T04:46:05.563011+0000","last_clean":"2024-04-26T04:46:05.563011+0000","last_became_active":"2024-04-26T04:45:22.646549+0000","last_became_peered":"2024-04-26T04:45:22.646549+0000","last_unstale":"2024-04-26T04:46:05.563011+0000","last_undegraded":"2024-04-26T04:46:05.563011+0000","last_fullsized":"2024-04-26T04:46:05.563011+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:14:26.381684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560585+0000","last_change":"2024-04-26T04:45:21.633696+0000","last_active":"2024-04-26T04:46:05.560585+0000","last_peered":"2024-04-26T04:46:05.560585+0000","last_clean":"2024-04-26T04:46:05.560585+0000","last_became_active":"2024-04-26T04:45:21.633431+0000","last_became_peered":"2024-04-26T04:45:21.633431+0000","last_unstale":"2024-04-26T04:46:05.560585+0000","last_undegraded":"2024-04-26T04:46:05.560585+0000","last_fullsized":"2024-04-26T04:46:05.560585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:23:57.941279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562553+0000","last_change":"2024-04-26T04:45:22.641284+0000","last_active":"2024-04-26T04:46:05.562553+0000","last_peered":"2024-04-26T04:46:05.562553+0000","last_clean":"2024-04-26T04:46:05.562553+0000","last_became_active":"2024-04-26T04:45:22.640982+0000","last_became_peered":"2024-04-26T04:45:22.640982+0000","last_unstale":"2024-04-26T04:46:05.562553+0000","last_undegraded":"2024-04-26T04:46:05.562553+0000","last_fullsized":"2024-04-26T04:46:05.562553+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:26:16.595756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559539+0000","last_change":"2024-04-26T04:45:21.642451+0000","last_active":"2024-04-26T04:46:05.559539+0000","last_peered":"2024-04-26T04:46:05.559539+0000","last_clean":"2024-04-26T04:46:05.559539+0000","last_became_active":"2024-04-26T04:45:21.641428+0000","last_became_peered":"2024-04-26T04:45:21.641428+0000","last_unstale":"2024-04-26T04:46:05.559539+0000","last_undegraded":"2024-04-26T04:46:05.559539+0000","last_fullsized":"2024-04-26T04:46:05.559539+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:43:10.838400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563001+0000","last_change":"2024-04-26T04:45:22.642636+0000","last_active":"2024-04-26T04:46:05.563001+0000","last_peered":"2024-04-26T04:46:05.563001+0000","last_clean":"2024-04-26T04:46:05.563001+0000","last_became_active":"2024-04-26T04:45:22.642325+0000","last_became_peered":"2024-04-26T04:45:22.642325+0000","last_unstale":"2024-04-26T04:46:05.563001+0000","last_undegraded":"2024-04-26T04:46:05.563001+0000","last_fullsized":"2024-04-26T04:46:05.563001+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:01:24.302695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562607+0000","last_change":"2024-04-26T04:45:21.643555+0000","last_active":"2024-04-26T04:46:05.562607+0000","last_peered":"2024-04-26T04:46:05.562607+0000","last_clean":"2024-04-26T04:46:05.562607+0000","last_became_active":"2024-04-26T04:45:21.643242+0000","last_became_peered":"2024-04-26T04:45:21.643242+0000","last_unstale":"2024-04-26T04:46:05.562607+0000","last_undegraded":"2024-04-26T04:46:05.562607+0000","last_fullsized":"2024-04-26T04:46:05.562607+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:07:39.491678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559253+0000","last_change":"2024-04-26T04:45:22.650094+0000","last_active":"2024-04-26T04:46:05.559253+0000","last_peered":"2024-04-26T04:46:05.559253+0000","last_clean":"2024-04-26T04:46:05.559253+0000","last_became_active":"2024-04-26T04:45:22.649627+0000","last_became_peered":"2024-04-26T04:45:22.649627+0000","last_unstale":"2024-04-26T04:46:05.559253+0000","last_undegraded":"2024-04-26T04:46:05.559253+0000","last_fullsized":"2024-04-26T04:46:05.559253+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:39:30.883085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560021+0000","last_change":"2024-04-26T04:45:21.632189+0000","last_active":"2024-04-26T04:46:05.560021+0000","last_peered":"2024-04-26T04:46:05.560021+0000","last_clean":"2024-04-26T04:46:05.560021+0000","last_became_active":"2024-04-26T04:45:21.631875+0000","last_became_peered":"2024-04-26T04:45:21.631875+0000","last_unstale":"2024-04-26T04:46:05.560021+0000","last_undegraded":"2024-04-26T04:46:05.560021+0000","last_fullsized":"2024-04-26T04:46:05.560021+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:02:56.126210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563414+0000","last_change":"2024-04-26T04:45:22.648545+0000","last_active":"2024-04-26T04:46:05.563414+0000","last_peered":"2024-04-26T04:46:05.563414+0000","last_clean":"2024-04-26T04:46:05.563414+0000","last_became_active":"2024-04-26T04:45:22.648249+0000","last_became_peered":"2024-04-26T04:45:22.648249+0000","last_unstale":"2024-04-26T04:46:05.563414+0000","last_undegraded":"2024-04-26T04:46:05.563414+0000","last_fullsized":"2024-04-26T04:46:05.563414+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:17:05.175726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916090+0000","last_change":"2024-04-26T04:45:21.629149+0000","last_active":"2024-04-26T04:46:05.916090+0000","last_peered":"2024-04-26T04:46:05.916090+0000","last_clean":"2024-04-26T04:46:05.916090+0000","last_became_active":"2024-04-26T04:45:21.628847+0000","last_became_peered":"2024-04-26T04:45:21.628847+0000","last_unstale":"2024-04-26T04:46:05.916090+0000","last_undegraded":"2024-04-26T04:46:05.916090+0000","last_fullsized":"2024-04-26T04:46:05.916090+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:03:05.754090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1316,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1347584,"data_stored":908605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.51900000000000002,"15min":0.52500000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.64300000000000002,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67800000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.65000000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.55600000000000005,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.66600000000000004,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.68799999999999994}]},{"osd":2,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72499999999999998,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.629},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53700000000000003,"15min":0.53500000000000003},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.75800000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42699999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.69499999999999995,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.39100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70699999999999996,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.71499999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.495},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70399999999999996,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52000000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.45700000000000002,"15min":0.45000000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.66600000000000004,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.52300000000000002,"15min":0.52500000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67600000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.55200000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52500000000000002,"15min":0.52800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66100000000000003,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70199999999999996,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.73399999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":907929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.52000000000000002,"15min":0.51800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.93300000000000005,"15min":1.121},"min":{"1min":0.38200000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67200000000000004,"5min":16.103999999999999,"15min":16.103999999999999},"last":0.59499999999999997}]},{"osd":1,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.81100000000000005,"15min":0.91800000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70999999999999996,"5min":9.75,"15min":9.75},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.42499999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78100000000000003,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.57399999999999995}]},{"osd":2,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52000000000000002,"15min":0.51900000000000002},"min":{"1min":0.38,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68000000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.76900000000000002,"15min":0.874},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.64700000000000002,"5min":9.7249999999999996,"15min":9.7249999999999996},"last":0.55100000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.44600000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.67100000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.95999999999999996,"15min":1.137},"min":{"1min":0.41999999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72199999999999998,"5min":16.128,"15min":16.128},"last":0.625}]},{"osd":4,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.42999999999999999,"15min":0.40999999999999998},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70499999999999996,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.48599999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.44700000000000001,"15min":0.434},"min":{"1min":0.34999999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68500000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.495,"15min":0.48099999999999998},"min":{"1min":0.36899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53900000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:45:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.51700000000000002,"15min":0.50700000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.47199999999999998,"15min":0.46200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.45500000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":316204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:18 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.36799999999999999,"15min":0.32800000000000001},"min":{"1min":0.27600000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.64700000000000002,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.34399999999999997},"min":{"1min":0.28699999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.41299999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:45:42 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.316},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38100000000000001},"min":{"1min":0.26800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.33400000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:46:04 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.48999999999999999,"15min":0.505},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61199999999999999,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.44500000000000001,"15min":0.46500000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56200000000000006,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.39500000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:45:24 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.58699999999999997,"15min":0.58099999999999996},"min":{"1min":0.47199999999999998,"5min":0.376,"15min":0.376},"max":{"1min":0.81699999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.73099999999999998,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.58599999999999997}]},{"osd":5,"last update":"Fri Apr 26 04:45:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.495,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76200000000000001,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58699999999999997,"15min":0.58499999999999996},"min":{"1min":0.52800000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.83899999999999997,"5min":0.874,"15min":0.874},"last":0.68700000000000006}]},{"osd":6,"last update":"Fri Apr 26 04:45:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.60599999999999998,"15min":0.60799999999999998},"min":{"1min":0.42699999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72299999999999998,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46000000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.79000000000000004,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.54500000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.64000000000000001,"15min":0.64200000000000002},"min":{"1min":0.48199999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.75600000000000001,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.532,"5min":0.83199999999999996,"15min":0.71599999999999997},"min":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":16.957999999999998,"5min":16.957999999999998,"15min":16.957999999999998},"last":0.60999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.376,"15min":0.35299999999999998},"min":{"1min":0.23000000000000001,"5min":0.192,"15min":0.192},"max":{"1min":0.46999999999999997,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.38800000000000001,"15min":0.34699999999999998},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.379}]},{"osd":2,"last update":"Fri Apr 26 04:45:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.40799999999999997,"15min":0.41099999999999998},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.60299999999999998,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.40899999999999997,"15min":0.39800000000000002},"min":{"1min":0.27600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.59099999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.27700000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.48499999999999999,"15min":0.49299999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.60999999999999999,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.48299999999999998,"15min":0.504},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.57899999999999996,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.36599999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58499999999999996,"15min":0.58199999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75900000000000001,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71999999999999997,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.496}]},{"osd":5,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.622,"15min":0.623},"min":{"1min":0.41099999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.876,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64400000000000002,"15min":0.64600000000000002},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.94599999999999995,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.50800000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.61599999999999999,"15min":0.61899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.65400000000000003,"15min":0.66900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.80800000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53200000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:45:37 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.46899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.79400000000000004,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85699999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:43 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.68600000000000005,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.60399999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:45:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.755,"5min":0.875,"15min":0.875},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.437,"15min":0.439},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67200000000000004,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.46899999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48699999999999999,"15min":0.47699999999999998},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.47899999999999998,"15min":0.48599999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.50900000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:45:24 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79600000000000004,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.624},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.59099999999999997,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83099999999999996,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.73799999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63,"15min":0.64400000000000002},"min":{"1min":0.45800000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.41199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.86499999999999999,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.66200000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.64600000000000002,"15min":0.65000000000000002},"min":{"1min":0.46600000000000003,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.68000000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:45:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.626,"15min":0.63100000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.58099999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:46:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.42699999999999999,"15min":0.40300000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.625,"5min":3.625,"15min":3.625},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.46500000000000002,"15min":0.44700000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.827,"5min":3.827,"15min":3.827},"last":0.32100000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:46:02 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.51800000000000002,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2000000000000002,"5min":7.2000000000000002,"15min":7.2000000000000002},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.52200000000000002,"15min":0.47599999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.1289999999999996,"5min":9.1289999999999996,"15min":9.1289999999999996},"last":0.48499999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:46:02 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.55300000000000005,"15min":0.51000000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.7759999999999998,"5min":8.7759999999999998,"15min":8.7759999999999998},"last":0.308},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.495,"15min":0.47299999999999998},"min":{"1min":0.27700000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.581,"5min":3.581,"15min":3.581},"last":0.44800000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:45:22 2024","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.61399999999999999,"15min":0.57999999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":3.6720000000000002,"5min":3.6720000000000002,"15min":3.6720000000000002},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.133,"5min":0.70299999999999996,"15min":0.63100000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":8.7110000000000003,"5min":8.7110000000000003,"15min":8.7110000000000003},"last":0.54800000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.65000000000000002,"15min":0.624},"min":{"1min":0.49099999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.4079999999999999,"5min":3.4079999999999999,"15min":3.4079999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.71399999999999997,"15min":0.65000000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.7449999999999992,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.60099999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.72299999999999998,"15min":0.67300000000000004},"min":{"1min":0.45900000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.6660000000000004,"5min":8.6660000000000004,"15min":8.6660000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.69099999999999995,"15min":0.66700000000000004},"min":{"1min":0.46400000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.71799999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.70399999999999996,"15min":0.67900000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":3.4820000000000002,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.67700000000000005,"15min":0.65100000000000002},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":3.722,"5min":3.722,"15min":3.722},"last":0.61299999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986225,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298088,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447064,"statfs":{"total":95995035648,"available":95689793536,"internally_reserved":0,"allocated":745472,"data_stored":317902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.52800000000000002,"15min":0.51600000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54000000000000004},"min":{"1min":0.40400000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.79200000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.49399999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52500000000000002,"15min":0.51600000000000001},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53700000000000003,"15min":0.53400000000000003},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73699999999999999,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.57599999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.55000000000000004,"15min":0.54500000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83299999999999996,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.66900000000000004}]},{"osd":3,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54300000000000004,"15min":0.53100000000000003},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78600000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.621},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.56299999999999994,"15min":0.55500000000000005},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.54300000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:45:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.41699999999999998,"15min":0.38700000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.64900000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.44},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.42899999999999999,"15min":0.39400000000000002},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.70799999999999996,"5min":0.88,"15min":0.88},"last":0.55400000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.44400000000000001,"15min":0.41899999999999998},"min":{"1min":0.29799999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.745,"5min":0.876,"15min":0.876},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44600000000000001,"15min":0.41799999999999998},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.66200000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.47499999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:45:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52200000000000002,"15min":0.51300000000000001},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52100000000000002,"15min":0.51600000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.68100000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54400000000000004,"15min":0.53900000000000003},"min":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.70099999999999996,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69699999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.35999999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.61699999999999999,"15min":0.56499999999999995},"min":{"1min":0.443,"5min":0.374,"15min":0.374},"max":{"1min":0.71999999999999997,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.629,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.48199999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.53100000000000003,"15min":0.52300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65300000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.61099999999999999,"15min":0.57899999999999996},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.5709999999999997,"5min":4.5709999999999997,"15min":4.5709999999999997},"last":0.54500000000000004}]},{"osd":3,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.63100000000000001,"15min":0.57699999999999996},"min":{"1min":0.42299999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71399999999999997,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56200000000000006,"15min":0.55000000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73899999999999999,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.503}]},{"osd":4,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.442,"15min":0.436},"min":{"1min":0.32100000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.46200000000000002,"15min":0.44700000000000001},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.47199999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:45:14 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.47799999999999998,"15min":0.47199999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.432},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.48599999999999999,"15min":0.47799999999999998},"min":{"1min":0.35599999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.58099999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.51700000000000002,"15min":0.51200000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.35299999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:46:22.032 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:46:22.301 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:22.703 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:22 smithi005 ceph-mon[36961]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:46:22.703 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:22 smithi005 ceph-mon[36961]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:22.703 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:22 smithi005 ceph-mon[26371]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:46:22.703 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:22 smithi005 ceph-mon[26371]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:22 smithi111 ceph-mon[30255]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:46:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:22 smithi111 ceph-mon[30255]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:24.359 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:46:24.360 INFO:teuthology.orchestra.run.smithi005.stderr:dumped all 2024-04-26T04:46:24.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:24 smithi005 ceph-mon[36961]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:24 smithi005 ceph-mon[26371]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:24 smithi111 ceph-mon[30255]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.130 INFO:teuthology.orchestra.run.smithi005.stdout:{"pg_ready":true,"pg_map":{"version":83,"stamp":"2024-04-26T04:46:23.499160+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386344,"kb_used_data":7476,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574872,"statfs":{"total":767960285184,"available":765516668928,"internally_reserved":0,"allocated":7655424,"data_stored":4308280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002694"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.915863+0000","last_change":"2024-04-26T04:45:22.644600+0000","last_active":"2024-04-26T04:46:05.915863+0000","last_peered":"2024-04-26T04:46:05.915863+0000","last_clean":"2024-04-26T04:46:05.915863+0000","last_became_active":"2024-04-26T04:45:22.644313+0000","last_became_peered":"2024-04-26T04:45:22.644313+0000","last_unstale":"2024-04-26T04:46:05.915863+0000","last_undegraded":"2024-04-26T04:46:05.915863+0000","last_fullsized":"2024-04-26T04:46:05.915863+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:46:58.521750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559643+0000","last_change":"2024-04-26T04:45:21.641941+0000","last_active":"2024-04-26T04:46:05.559643+0000","last_peered":"2024-04-26T04:46:05.559643+0000","last_clean":"2024-04-26T04:46:05.559643+0000","last_became_active":"2024-04-26T04:45:21.639163+0000","last_became_peered":"2024-04-26T04:45:21.639163+0000","last_unstale":"2024-04-26T04:46:05.559643+0000","last_undegraded":"2024-04-26T04:46:05.559643+0000","last_fullsized":"2024-04-26T04:46:05.559643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:15:16.779001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562946+0000","last_change":"2024-04-26T04:45:22.651667+0000","last_active":"2024-04-26T04:46:05.562946+0000","last_peered":"2024-04-26T04:46:05.562946+0000","last_clean":"2024-04-26T04:46:05.562946+0000","last_became_active":"2024-04-26T04:45:22.651398+0000","last_became_peered":"2024-04-26T04:45:22.651398+0000","last_unstale":"2024-04-26T04:46:05.562946+0000","last_undegraded":"2024-04-26T04:46:05.562946+0000","last_fullsized":"2024-04-26T04:46:05.562946+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:31:57.490271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563741+0000","last_change":"2024-04-26T04:45:21.637069+0000","last_active":"2024-04-26T04:46:05.563741+0000","last_peered":"2024-04-26T04:46:05.563741+0000","last_clean":"2024-04-26T04:46:05.563741+0000","last_became_active":"2024-04-26T04:45:21.636722+0000","last_became_peered":"2024-04-26T04:45:21.636722+0000","last_unstale":"2024-04-26T04:46:05.563741+0000","last_undegraded":"2024-04-26T04:46:05.563741+0000","last_fullsized":"2024-04-26T04:46:05.563741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:14:27.222222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916090+0000","last_change":"2024-04-26T04:45:22.648085+0000","last_active":"2024-04-26T04:46:05.916090+0000","last_peered":"2024-04-26T04:46:05.916090+0000","last_clean":"2024-04-26T04:46:05.916090+0000","last_became_active":"2024-04-26T04:45:22.647782+0000","last_became_peered":"2024-04-26T04:45:22.647782+0000","last_unstale":"2024-04-26T04:46:05.916090+0000","last_undegraded":"2024-04-26T04:46:05.916090+0000","last_fullsized":"2024-04-26T04:46:05.916090+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:37:12.680619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560575+0000","last_change":"2024-04-26T04:45:21.634660+0000","last_active":"2024-04-26T04:46:05.560575+0000","last_peered":"2024-04-26T04:46:05.560575+0000","last_clean":"2024-04-26T04:46:05.560575+0000","last_became_active":"2024-04-26T04:45:21.634404+0000","last_became_peered":"2024-04-26T04:45:21.634404+0000","last_unstale":"2024-04-26T04:46:05.560575+0000","last_undegraded":"2024-04-26T04:46:05.560575+0000","last_fullsized":"2024-04-26T04:46:05.560575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:28:20.390001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916482+0000","last_change":"2024-04-26T04:45:22.648270+0000","last_active":"2024-04-26T04:46:05.916482+0000","last_peered":"2024-04-26T04:46:05.916482+0000","last_clean":"2024-04-26T04:46:05.916482+0000","last_became_active":"2024-04-26T04:45:22.647940+0000","last_became_peered":"2024-04-26T04:45:22.647940+0000","last_unstale":"2024-04-26T04:46:05.916482+0000","last_undegraded":"2024-04-26T04:46:05.916482+0000","last_fullsized":"2024-04-26T04:46:05.916482+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:14:16.459554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560487+0000","last_change":"2024-04-26T04:45:21.641206+0000","last_active":"2024-04-26T04:46:05.560487+0000","last_peered":"2024-04-26T04:46:05.560487+0000","last_clean":"2024-04-26T04:46:05.560487+0000","last_became_active":"2024-04-26T04:45:21.640887+0000","last_became_peered":"2024-04-26T04:45:21.640887+0000","last_unstale":"2024-04-26T04:46:05.560487+0000","last_undegraded":"2024-04-26T04:46:05.560487+0000","last_fullsized":"2024-04-26T04:46:05.560487+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:04:06.041378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:46:06.882564+0000","last_change":"2024-04-26T04:46:06.882564+0000","last_active":"2024-04-26T04:46:06.882564+0000","last_peered":"2024-04-26T04:46:06.882564+0000","last_clean":"2024-04-26T04:46:06.882564+0000","last_became_active":"2024-04-26T04:46:06.882204+0000","last_became_peered":"2024-04-26T04:46:06.882204+0000","last_unstale":"2024-04-26T04:46:06.882564+0000","last_undegraded":"2024-04-26T04:46:06.882564+0000","last_fullsized":"2024-04-26T04:46:06.882564+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:06:15.186341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559264+0000","last_change":"2024-04-26T04:45:21.631557+0000","last_active":"2024-04-26T04:46:05.559264+0000","last_peered":"2024-04-26T04:46:05.559264+0000","last_clean":"2024-04-26T04:46:05.559264+0000","last_became_active":"2024-04-26T04:45:21.630609+0000","last_became_peered":"2024-04-26T04:45:21.630609+0000","last_unstale":"2024-04-26T04:46:05.559264+0000","last_undegraded":"2024-04-26T04:46:05.559264+0000","last_fullsized":"2024-04-26T04:46:05.559264+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:14:38.394486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916088+0000","last_change":"2024-04-26T04:45:22.644110+0000","last_active":"2024-04-26T04:46:05.916088+0000","last_peered":"2024-04-26T04:46:05.916088+0000","last_clean":"2024-04-26T04:46:05.916088+0000","last_became_active":"2024-04-26T04:45:22.643811+0000","last_became_peered":"2024-04-26T04:45:22.643811+0000","last_unstale":"2024-04-26T04:46:05.916088+0000","last_undegraded":"2024-04-26T04:46:05.916088+0000","last_fullsized":"2024-04-26T04:46:05.916088+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:20:05.112184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559335+0000","last_change":"2024-04-26T04:45:21.625983+0000","last_active":"2024-04-26T04:46:05.559335+0000","last_peered":"2024-04-26T04:46:05.559335+0000","last_clean":"2024-04-26T04:46:05.559335+0000","last_became_active":"2024-04-26T04:45:21.625586+0000","last_became_peered":"2024-04-26T04:45:21.625586+0000","last_unstale":"2024-04-26T04:46:05.559335+0000","last_undegraded":"2024-04-26T04:46:05.559335+0000","last_fullsized":"2024-04-26T04:46:05.559335+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:46:48.199296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563104+0000","last_change":"2024-04-26T04:44:08.479180+0000","last_active":"2024-04-26T04:46:05.563104+0000","last_peered":"2024-04-26T04:46:05.563104+0000","last_clean":"2024-04-26T04:46:05.563104+0000","last_became_active":"2024-04-26T04:44:08.478822+0000","last_became_peered":"2024-04-26T04:44:08.478822+0000","last_unstale":"2024-04-26T04:46:05.563104+0000","last_undegraded":"2024-04-26T04:46:05.563104+0000","last_fullsized":"2024-04-26T04:46:05.563104+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:08:10.312422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562442+0000","last_change":"2024-04-26T04:45:22.648080+0000","last_active":"2024-04-26T04:46:05.562442+0000","last_peered":"2024-04-26T04:46:05.562442+0000","last_clean":"2024-04-26T04:46:05.562442+0000","last_became_active":"2024-04-26T04:45:22.647748+0000","last_became_peered":"2024-04-26T04:45:22.647748+0000","last_unstale":"2024-04-26T04:46:05.562442+0000","last_undegraded":"2024-04-26T04:46:05.562442+0000","last_fullsized":"2024-04-26T04:46:05.562442+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:05:47.555778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560719+0000","last_change":"2024-04-26T04:44:08.965639+0000","last_active":"2024-04-26T04:46:05.560719+0000","last_peered":"2024-04-26T04:46:05.560719+0000","last_clean":"2024-04-26T04:46:05.560719+0000","last_became_active":"2024-04-26T04:44:08.965296+0000","last_became_peered":"2024-04-26T04:44:08.965296+0000","last_unstale":"2024-04-26T04:46:05.560719+0000","last_undegraded":"2024-04-26T04:46:05.560719+0000","last_fullsized":"2024-04-26T04:46:05.560719+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:17:21.026489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563799+0000","last_change":"2024-04-26T04:45:21.645833+0000","last_active":"2024-04-26T04:46:05.563799+0000","last_peered":"2024-04-26T04:46:05.563799+0000","last_clean":"2024-04-26T04:46:05.563799+0000","last_became_active":"2024-04-26T04:45:21.645542+0000","last_became_peered":"2024-04-26T04:45:21.645542+0000","last_unstale":"2024-04-26T04:46:05.563799+0000","last_undegraded":"2024-04-26T04:46:05.563799+0000","last_fullsized":"2024-04-26T04:46:05.563799+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:44:25.617808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559183+0000","last_change":"2024-04-26T04:45:22.642805+0000","last_active":"2024-04-26T04:46:05.559183+0000","last_peered":"2024-04-26T04:46:05.559183+0000","last_clean":"2024-04-26T04:46:05.559183+0000","last_became_active":"2024-04-26T04:45:22.642314+0000","last_became_peered":"2024-04-26T04:45:22.642314+0000","last_unstale":"2024-04-26T04:46:05.559183+0000","last_undegraded":"2024-04-26T04:46:05.559183+0000","last_fullsized":"2024-04-26T04:46:05.559183+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:10:38.308347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.561073+0000","last_change":"2024-04-26T04:45:21.636871+0000","last_active":"2024-04-26T04:46:05.561073+0000","last_peered":"2024-04-26T04:46:05.561073+0000","last_clean":"2024-04-26T04:46:05.561073+0000","last_became_active":"2024-04-26T04:45:21.636389+0000","last_became_peered":"2024-04-26T04:45:21.636389+0000","last_unstale":"2024-04-26T04:46:05.561073+0000","last_undegraded":"2024-04-26T04:46:05.561073+0000","last_fullsized":"2024-04-26T04:46:05.561073+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:30:22.877766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565470+0000","last_change":"2024-04-26T04:44:08.965486+0000","last_active":"2024-04-26T04:46:05.565470+0000","last_peered":"2024-04-26T04:46:05.565470+0000","last_clean":"2024-04-26T04:46:05.565470+0000","last_became_active":"2024-04-26T04:44:08.965051+0000","last_became_peered":"2024-04-26T04:44:08.965051+0000","last_unstale":"2024-04-26T04:46:05.565470+0000","last_undegraded":"2024-04-26T04:46:05.565470+0000","last_fullsized":"2024-04-26T04:46:05.565470+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:47:18.277374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559353+0000","last_change":"2024-04-26T04:45:22.641594+0000","last_active":"2024-04-26T04:46:05.559353+0000","last_peered":"2024-04-26T04:46:05.559353+0000","last_clean":"2024-04-26T04:46:05.559353+0000","last_became_active":"2024-04-26T04:45:22.641288+0000","last_became_peered":"2024-04-26T04:45:22.641288+0000","last_unstale":"2024-04-26T04:46:05.559353+0000","last_undegraded":"2024-04-26T04:46:05.559353+0000","last_fullsized":"2024-04-26T04:46:05.559353+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:38:39.016111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.558932+0000","last_change":"2024-04-26T04:45:21.639327+0000","last_active":"2024-04-26T04:46:05.558932+0000","last_peered":"2024-04-26T04:46:05.558932+0000","last_clean":"2024-04-26T04:46:05.558932+0000","last_became_active":"2024-04-26T04:45:21.638898+0000","last_became_peered":"2024-04-26T04:45:21.638898+0000","last_unstale":"2024-04-26T04:46:05.558932+0000","last_undegraded":"2024-04-26T04:46:05.558932+0000","last_fullsized":"2024-04-26T04:46:05.558932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:15:11.955760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565195+0000","last_change":"2024-04-26T04:44:08.481145+0000","last_active":"2024-04-26T04:46:05.565195+0000","last_peered":"2024-04-26T04:46:05.565195+0000","last_clean":"2024-04-26T04:46:05.565195+0000","last_became_active":"2024-04-26T04:44:08.480765+0000","last_became_peered":"2024-04-26T04:44:08.480765+0000","last_unstale":"2024-04-26T04:46:05.565195+0000","last_undegraded":"2024-04-26T04:46:05.565195+0000","last_fullsized":"2024-04-26T04:46:05.565195+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:48:08.184965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563055+0000","last_change":"2024-04-26T04:45:22.645366+0000","last_active":"2024-04-26T04:46:05.563055+0000","last_peered":"2024-04-26T04:46:05.563055+0000","last_clean":"2024-04-26T04:46:05.563055+0000","last_became_active":"2024-04-26T04:45:22.645130+0000","last_became_peered":"2024-04-26T04:45:22.645130+0000","last_unstale":"2024-04-26T04:46:05.563055+0000","last_undegraded":"2024-04-26T04:46:05.563055+0000","last_fullsized":"2024-04-26T04:46:05.563055+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:52:52.129108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562606+0000","last_change":"2024-04-26T04:45:21.637077+0000","last_active":"2024-04-26T04:46:05.562606+0000","last_peered":"2024-04-26T04:46:05.562606+0000","last_clean":"2024-04-26T04:46:05.562606+0000","last_became_active":"2024-04-26T04:45:21.636047+0000","last_became_peered":"2024-04-26T04:45:21.636047+0000","last_unstale":"2024-04-26T04:46:05.562606+0000","last_undegraded":"2024-04-26T04:46:05.562606+0000","last_fullsized":"2024-04-26T04:46:05.562606+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:53:59.428390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":278,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565171+0000","last_change":"2024-04-26T04:43:35.916221+0000","last_active":"2024-04-26T04:46:05.565171+0000","last_peered":"2024-04-26T04:46:05.565171+0000","last_clean":"2024-04-26T04:46:05.565171+0000","last_became_active":"2024-04-26T04:43:35.915728+0000","last_became_peered":"2024-04-26T04:43:35.915728+0000","last_unstale":"2024-04-26T04:46:05.565171+0000","last_undegraded":"2024-04-26T04:46:05.565171+0000","last_fullsized":"2024-04-26T04:46:05.565171+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_clean_scrub_stamp":"2024-04-26T04:41:33.492763+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:47:26.279649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560529+0000","last_change":"2024-04-26T04:44:08.479414+0000","last_active":"2024-04-26T04:46:05.560529+0000","last_peered":"2024-04-26T04:46:05.560529+0000","last_clean":"2024-04-26T04:46:05.560529+0000","last_became_active":"2024-04-26T04:44:08.478584+0000","last_became_peered":"2024-04-26T04:44:08.478584+0000","last_unstale":"2024-04-26T04:46:05.560529+0000","last_undegraded":"2024-04-26T04:46:05.560529+0000","last_fullsized":"2024-04-26T04:46:05.560529+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:02:29.012874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560401+0000","last_change":"2024-04-26T04:45:22.640395+0000","last_active":"2024-04-26T04:46:05.560401+0000","last_peered":"2024-04-26T04:46:05.560401+0000","last_clean":"2024-04-26T04:46:05.560401+0000","last_became_active":"2024-04-26T04:45:22.640113+0000","last_became_peered":"2024-04-26T04:45:22.640113+0000","last_unstale":"2024-04-26T04:46:05.560401+0000","last_undegraded":"2024-04-26T04:46:05.560401+0000","last_fullsized":"2024-04-26T04:46:05.560401+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:58:17.399345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559817+0000","last_change":"2024-04-26T04:45:21.628785+0000","last_active":"2024-04-26T04:46:05.559817+0000","last_peered":"2024-04-26T04:46:05.559817+0000","last_clean":"2024-04-26T04:46:05.559817+0000","last_became_active":"2024-04-26T04:45:21.628460+0000","last_became_peered":"2024-04-26T04:45:21.628460+0000","last_unstale":"2024-04-26T04:46:05.559817+0000","last_undegraded":"2024-04-26T04:46:05.559817+0000","last_fullsized":"2024-04-26T04:46:05.559817+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:05:42.764810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562756+0000","last_change":"2024-04-26T04:44:09.231105+0000","last_active":"2024-04-26T04:46:05.562756+0000","last_peered":"2024-04-26T04:46:05.562756+0000","last_clean":"2024-04-26T04:46:05.562756+0000","last_became_active":"2024-04-26T04:44:09.230737+0000","last_became_peered":"2024-04-26T04:44:09.230737+0000","last_unstale":"2024-04-26T04:46:05.562756+0000","last_undegraded":"2024-04-26T04:46:05.562756+0000","last_fullsized":"2024-04-26T04:46:05.562756+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:25:04.219657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.561030+0000","last_change":"2024-04-26T04:45:22.644736+0000","last_active":"2024-04-26T04:46:05.561030+0000","last_peered":"2024-04-26T04:46:05.561030+0000","last_clean":"2024-04-26T04:46:05.561030+0000","last_became_active":"2024-04-26T04:45:22.644457+0000","last_became_peered":"2024-04-26T04:45:22.644457+0000","last_unstale":"2024-04-26T04:46:05.561030+0000","last_undegraded":"2024-04-26T04:46:05.561030+0000","last_fullsized":"2024-04-26T04:46:05.561030+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:42:51.554720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916199+0000","last_change":"2024-04-26T04:45:21.636903+0000","last_active":"2024-04-26T04:46:05.916199+0000","last_peered":"2024-04-26T04:46:05.916199+0000","last_clean":"2024-04-26T04:46:05.916199+0000","last_became_active":"2024-04-26T04:45:21.635678+0000","last_became_peered":"2024-04-26T04:45:21.635678+0000","last_unstale":"2024-04-26T04:46:05.916199+0000","last_undegraded":"2024-04-26T04:46:05.916199+0000","last_fullsized":"2024-04-26T04:46:05.916199+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:17:03.267381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560419+0000","last_change":"2024-04-26T04:44:08.477772+0000","last_active":"2024-04-26T04:46:05.560419+0000","last_peered":"2024-04-26T04:46:05.560419+0000","last_clean":"2024-04-26T04:46:05.560419+0000","last_became_active":"2024-04-26T04:44:08.477431+0000","last_became_peered":"2024-04-26T04:44:08.477431+0000","last_unstale":"2024-04-26T04:46:05.560419+0000","last_undegraded":"2024-04-26T04:46:05.560419+0000","last_fullsized":"2024-04-26T04:46:05.560419+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:50:07.493196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560354+0000","last_change":"2024-04-26T04:45:22.642296+0000","last_active":"2024-04-26T04:46:05.560354+0000","last_peered":"2024-04-26T04:46:05.560354+0000","last_clean":"2024-04-26T04:46:05.560354+0000","last_became_active":"2024-04-26T04:45:22.642060+0000","last_became_peered":"2024-04-26T04:45:22.642060+0000","last_unstale":"2024-04-26T04:46:05.560354+0000","last_undegraded":"2024-04-26T04:46:05.560354+0000","last_fullsized":"2024-04-26T04:46:05.560354+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:19:16.633439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560510+0000","last_change":"2024-04-26T04:45:21.636592+0000","last_active":"2024-04-26T04:46:05.560510+0000","last_peered":"2024-04-26T04:46:05.560510+0000","last_clean":"2024-04-26T04:46:05.560510+0000","last_became_active":"2024-04-26T04:45:21.636257+0000","last_became_peered":"2024-04-26T04:45:21.636257+0000","last_unstale":"2024-04-26T04:46:05.560510+0000","last_undegraded":"2024-04-26T04:46:05.560510+0000","last_fullsized":"2024-04-26T04:46:05.560510+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:28:18.062261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560935+0000","last_change":"2024-04-26T04:44:08.480906+0000","last_active":"2024-04-26T04:46:05.560935+0000","last_peered":"2024-04-26T04:46:05.560935+0000","last_clean":"2024-04-26T04:46:05.560935+0000","last_became_active":"2024-04-26T04:44:08.480613+0000","last_became_peered":"2024-04-26T04:44:08.480613+0000","last_unstale":"2024-04-26T04:46:05.560935+0000","last_undegraded":"2024-04-26T04:46:05.560935+0000","last_fullsized":"2024-04-26T04:46:05.560935+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:20:31.237343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560487+0000","last_change":"2024-04-26T04:45:22.643691+0000","last_active":"2024-04-26T04:46:05.560487+0000","last_peered":"2024-04-26T04:46:05.560487+0000","last_clean":"2024-04-26T04:46:05.560487+0000","last_became_active":"2024-04-26T04:45:22.643281+0000","last_became_peered":"2024-04-26T04:45:22.643281+0000","last_unstale":"2024-04-26T04:46:05.560487+0000","last_undegraded":"2024-04-26T04:46:05.560487+0000","last_fullsized":"2024-04-26T04:46:05.560487+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:34:41.476802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562456+0000","last_change":"2024-04-26T04:45:21.645718+0000","last_active":"2024-04-26T04:46:05.562456+0000","last_peered":"2024-04-26T04:46:05.562456+0000","last_clean":"2024-04-26T04:46:05.562456+0000","last_became_active":"2024-04-26T04:45:21.645368+0000","last_became_peered":"2024-04-26T04:45:21.645368+0000","last_unstale":"2024-04-26T04:46:05.562456+0000","last_undegraded":"2024-04-26T04:46:05.562456+0000","last_fullsized":"2024-04-26T04:46:05.562456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:41:26.200426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916354+0000","last_change":"2024-04-26T04:45:22.643766+0000","last_active":"2024-04-26T04:46:05.916354+0000","last_peered":"2024-04-26T04:46:05.916354+0000","last_clean":"2024-04-26T04:46:05.916354+0000","last_became_active":"2024-04-26T04:45:22.643486+0000","last_became_peered":"2024-04-26T04:45:22.643486+0000","last_unstale":"2024-04-26T04:46:05.916354+0000","last_undegraded":"2024-04-26T04:46:05.916354+0000","last_fullsized":"2024-04-26T04:46:05.916354+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:04:44.877539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916126+0000","last_change":"2024-04-26T04:45:21.635507+0000","last_active":"2024-04-26T04:46:05.916126+0000","last_peered":"2024-04-26T04:46:05.916126+0000","last_clean":"2024-04-26T04:46:05.916126+0000","last_became_active":"2024-04-26T04:45:21.634530+0000","last_became_peered":"2024-04-26T04:45:21.634530+0000","last_unstale":"2024-04-26T04:46:05.916126+0000","last_undegraded":"2024-04-26T04:46:05.916126+0000","last_fullsized":"2024-04-26T04:46:05.916126+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:36:40.790413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916519+0000","last_change":"2024-04-26T04:45:22.644087+0000","last_active":"2024-04-26T04:46:05.916519+0000","last_peered":"2024-04-26T04:46:05.916519+0000","last_clean":"2024-04-26T04:46:05.916519+0000","last_became_active":"2024-04-26T04:45:22.643763+0000","last_became_peered":"2024-04-26T04:45:22.643763+0000","last_unstale":"2024-04-26T04:46:05.916519+0000","last_undegraded":"2024-04-26T04:46:05.916519+0000","last_fullsized":"2024-04-26T04:46:05.916519+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:20:27.062806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562841+0000","last_change":"2024-04-26T04:45:21.636924+0000","last_active":"2024-04-26T04:46:05.562841+0000","last_peered":"2024-04-26T04:46:05.562841+0000","last_clean":"2024-04-26T04:46:05.562841+0000","last_became_active":"2024-04-26T04:45:21.636646+0000","last_became_peered":"2024-04-26T04:45:21.636646+0000","last_unstale":"2024-04-26T04:46:05.562841+0000","last_undegraded":"2024-04-26T04:46:05.562841+0000","last_fullsized":"2024-04-26T04:46:05.562841+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:45:22.147993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916221+0000","last_change":"2024-04-26T04:45:22.648565+0000","last_active":"2024-04-26T04:46:05.916221+0000","last_peered":"2024-04-26T04:46:05.916221+0000","last_clean":"2024-04-26T04:46:05.916221+0000","last_became_active":"2024-04-26T04:45:22.647969+0000","last_became_peered":"2024-04-26T04:45:22.647969+0000","last_unstale":"2024-04-26T04:46:05.916221+0000","last_undegraded":"2024-04-26T04:46:05.916221+0000","last_fullsized":"2024-04-26T04:46:05.916221+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:11.200395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563873+0000","last_change":"2024-04-26T04:45:21.645388+0000","last_active":"2024-04-26T04:46:05.563873+0000","last_peered":"2024-04-26T04:46:05.563873+0000","last_clean":"2024-04-26T04:46:05.563873+0000","last_became_active":"2024-04-26T04:45:21.645053+0000","last_became_peered":"2024-04-26T04:45:21.645053+0000","last_unstale":"2024-04-26T04:46:05.563873+0000","last_undegraded":"2024-04-26T04:46:05.563873+0000","last_fullsized":"2024-04-26T04:46:05.563873+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:38:37.713482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559109+0000","last_change":"2024-04-26T04:45:22.642590+0000","last_active":"2024-04-26T04:46:05.559109+0000","last_peered":"2024-04-26T04:46:05.559109+0000","last_clean":"2024-04-26T04:46:05.559109+0000","last_became_active":"2024-04-26T04:45:22.641884+0000","last_became_peered":"2024-04-26T04:45:22.641884+0000","last_unstale":"2024-04-26T04:46:05.559109+0000","last_undegraded":"2024-04-26T04:46:05.559109+0000","last_fullsized":"2024-04-26T04:46:05.559109+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:35:53.039625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560487+0000","last_change":"2024-04-26T04:45:21.631299+0000","last_active":"2024-04-26T04:46:05.560487+0000","last_peered":"2024-04-26T04:46:05.560487+0000","last_clean":"2024-04-26T04:46:05.560487+0000","last_became_active":"2024-04-26T04:45:21.630990+0000","last_became_peered":"2024-04-26T04:45:21.630990+0000","last_unstale":"2024-04-26T04:46:05.560487+0000","last_undegraded":"2024-04-26T04:46:05.560487+0000","last_fullsized":"2024-04-26T04:46:05.560487+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:36:19.517763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:46:06.568711+0000","last_change":"2024-04-26T04:46:06.568711+0000","last_active":"2024-04-26T04:46:06.568711+0000","last_peered":"2024-04-26T04:46:06.568711+0000","last_clean":"2024-04-26T04:46:06.568711+0000","last_became_active":"2024-04-26T04:46:06.568385+0000","last_became_peered":"2024-04-26T04:46:06.568385+0000","last_unstale":"2024-04-26T04:46:06.568711+0000","last_undegraded":"2024-04-26T04:46:06.568711+0000","last_fullsized":"2024-04-26T04:46:06.568711+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:25:21.170929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565572+0000","last_change":"2024-04-26T04:45:21.638699+0000","last_active":"2024-04-26T04:46:05.565572+0000","last_peered":"2024-04-26T04:46:05.565572+0000","last_clean":"2024-04-26T04:46:05.565572+0000","last_became_active":"2024-04-26T04:45:21.638390+0000","last_became_peered":"2024-04-26T04:45:21.638390+0000","last_unstale":"2024-04-26T04:46:05.565572+0000","last_undegraded":"2024-04-26T04:46:05.565572+0000","last_fullsized":"2024-04-26T04:46:05.565572+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:29:59.647984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.915900+0000","last_change":"2024-04-26T04:45:22.648384+0000","last_active":"2024-04-26T04:46:05.915900+0000","last_peered":"2024-04-26T04:46:05.915900+0000","last_clean":"2024-04-26T04:46:05.915900+0000","last_became_active":"2024-04-26T04:45:22.647989+0000","last_became_peered":"2024-04-26T04:45:22.647989+0000","last_unstale":"2024-04-26T04:46:05.915900+0000","last_undegraded":"2024-04-26T04:46:05.915900+0000","last_fullsized":"2024-04-26T04:46:05.915900+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:43:56.351981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565661+0000","last_change":"2024-04-26T04:45:21.643690+0000","last_active":"2024-04-26T04:46:05.565661+0000","last_peered":"2024-04-26T04:46:05.565661+0000","last_clean":"2024-04-26T04:46:05.565661+0000","last_became_active":"2024-04-26T04:45:21.643239+0000","last_became_peered":"2024-04-26T04:45:21.643239+0000","last_unstale":"2024-04-26T04:46:05.565661+0000","last_undegraded":"2024-04-26T04:46:05.565661+0000","last_fullsized":"2024-04-26T04:46:05.565661+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:47:09.366443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560512+0000","last_change":"2024-04-26T04:45:22.642459+0000","last_active":"2024-04-26T04:46:05.560512+0000","last_peered":"2024-04-26T04:46:05.560512+0000","last_clean":"2024-04-26T04:46:05.560512+0000","last_became_active":"2024-04-26T04:45:22.642177+0000","last_became_peered":"2024-04-26T04:45:22.642177+0000","last_unstale":"2024-04-26T04:46:05.560512+0000","last_undegraded":"2024-04-26T04:46:05.560512+0000","last_fullsized":"2024-04-26T04:46:05.560512+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:14.494066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565649+0000","last_change":"2024-04-26T04:45:21.634985+0000","last_active":"2024-04-26T04:46:05.565649+0000","last_peered":"2024-04-26T04:46:05.565649+0000","last_clean":"2024-04-26T04:46:05.565649+0000","last_became_active":"2024-04-26T04:45:21.634663+0000","last_became_peered":"2024-04-26T04:45:21.634663+0000","last_unstale":"2024-04-26T04:46:05.565649+0000","last_undegraded":"2024-04-26T04:46:05.565649+0000","last_fullsized":"2024-04-26T04:46:05.565649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:30:52.426028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563681+0000","last_change":"2024-04-26T04:45:22.647497+0000","last_active":"2024-04-26T04:46:05.563681+0000","last_peered":"2024-04-26T04:46:05.563681+0000","last_clean":"2024-04-26T04:46:05.563681+0000","last_became_active":"2024-04-26T04:45:22.647217+0000","last_became_peered":"2024-04-26T04:45:22.647217+0000","last_unstale":"2024-04-26T04:46:05.563681+0000","last_undegraded":"2024-04-26T04:46:05.563681+0000","last_fullsized":"2024-04-26T04:46:05.563681+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:54:55.108116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562726+0000","last_change":"2024-04-26T04:45:21.639205+0000","last_active":"2024-04-26T04:46:05.562726+0000","last_peered":"2024-04-26T04:46:05.562726+0000","last_clean":"2024-04-26T04:46:05.562726+0000","last_became_active":"2024-04-26T04:45:21.638924+0000","last_became_peered":"2024-04-26T04:45:21.638924+0000","last_unstale":"2024-04-26T04:46:05.562726+0000","last_undegraded":"2024-04-26T04:46:05.562726+0000","last_fullsized":"2024-04-26T04:46:05.562726+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:29:07.972721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559184+0000","last_change":"2024-04-26T04:45:22.650422+0000","last_active":"2024-04-26T04:46:05.559184+0000","last_peered":"2024-04-26T04:46:05.559184+0000","last_clean":"2024-04-26T04:46:05.559184+0000","last_became_active":"2024-04-26T04:45:22.650089+0000","last_became_peered":"2024-04-26T04:45:22.650089+0000","last_unstale":"2024-04-26T04:46:05.559184+0000","last_undegraded":"2024-04-26T04:46:05.559184+0000","last_fullsized":"2024-04-26T04:46:05.559184+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:38:42.148255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565122+0000","last_change":"2024-04-26T04:45:21.634430+0000","last_active":"2024-04-26T04:46:05.565122+0000","last_peered":"2024-04-26T04:46:05.565122+0000","last_clean":"2024-04-26T04:46:05.565122+0000","last_became_active":"2024-04-26T04:45:21.634149+0000","last_became_peered":"2024-04-26T04:45:21.634149+0000","last_unstale":"2024-04-26T04:46:05.565122+0000","last_undegraded":"2024-04-26T04:46:05.565122+0000","last_fullsized":"2024-04-26T04:46:05.565122+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:32.326287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560432+0000","last_change":"2024-04-26T04:45:22.642712+0000","last_active":"2024-04-26T04:46:05.560432+0000","last_peered":"2024-04-26T04:46:05.560432+0000","last_clean":"2024-04-26T04:46:05.560432+0000","last_became_active":"2024-04-26T04:45:22.641574+0000","last_became_peered":"2024-04-26T04:45:22.641574+0000","last_unstale":"2024-04-26T04:46:05.560432+0000","last_undegraded":"2024-04-26T04:46:05.560432+0000","last_fullsized":"2024-04-26T04:46:05.560432+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:26:03.339050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559083+0000","last_change":"2024-04-26T04:45:21.640861+0000","last_active":"2024-04-26T04:46:05.559083+0000","last_peered":"2024-04-26T04:46:05.559083+0000","last_clean":"2024-04-26T04:46:05.559083+0000","last_became_active":"2024-04-26T04:45:21.639828+0000","last_became_peered":"2024-04-26T04:45:21.639828+0000","last_unstale":"2024-04-26T04:46:05.559083+0000","last_undegraded":"2024-04-26T04:46:05.559083+0000","last_fullsized":"2024-04-26T04:46:05.559083+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:04:14.253122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:46:06.568107+0000","last_change":"2024-04-26T04:46:06.568107+0000","last_active":"2024-04-26T04:46:06.568107+0000","last_peered":"2024-04-26T04:46:06.568107+0000","last_clean":"2024-04-26T04:46:06.568107+0000","last_became_active":"2024-04-26T04:46:06.567748+0000","last_became_peered":"2024-04-26T04:46:06.567748+0000","last_unstale":"2024-04-26T04:46:06.568107+0000","last_undegraded":"2024-04-26T04:46:06.568107+0000","last_fullsized":"2024-04-26T04:46:06.568107+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:55:12.034909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565428+0000","last_change":"2024-04-26T04:45:21.643210+0000","last_active":"2024-04-26T04:46:05.565428+0000","last_peered":"2024-04-26T04:46:05.565428+0000","last_clean":"2024-04-26T04:46:05.565428+0000","last_became_active":"2024-04-26T04:45:21.642825+0000","last_became_peered":"2024-04-26T04:45:21.642825+0000","last_unstale":"2024-04-26T04:46:05.565428+0000","last_undegraded":"2024-04-26T04:46:05.565428+0000","last_fullsized":"2024-04-26T04:46:05.565428+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:52:17.053860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916344+0000","last_change":"2024-04-26T04:45:22.647005+0000","last_active":"2024-04-26T04:46:05.916344+0000","last_peered":"2024-04-26T04:46:05.916344+0000","last_clean":"2024-04-26T04:46:05.916344+0000","last_became_active":"2024-04-26T04:45:22.646753+0000","last_became_peered":"2024-04-26T04:45:22.646753+0000","last_unstale":"2024-04-26T04:46:05.916344+0000","last_undegraded":"2024-04-26T04:46:05.916344+0000","last_fullsized":"2024-04-26T04:46:05.916344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:22:11.999704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916285+0000","last_change":"2024-04-26T04:45:21.631617+0000","last_active":"2024-04-26T04:46:05.916285+0000","last_peered":"2024-04-26T04:46:05.916285+0000","last_clean":"2024-04-26T04:46:05.916285+0000","last_became_active":"2024-04-26T04:45:21.631218+0000","last_became_peered":"2024-04-26T04:45:21.631218+0000","last_unstale":"2024-04-26T04:46:05.916285+0000","last_undegraded":"2024-04-26T04:46:05.916285+0000","last_fullsized":"2024-04-26T04:46:05.916285+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:50:12.166307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560161+0000","last_change":"2024-04-26T04:45:22.644166+0000","last_active":"2024-04-26T04:46:05.560161+0000","last_peered":"2024-04-26T04:46:05.560161+0000","last_clean":"2024-04-26T04:46:05.560161+0000","last_became_active":"2024-04-26T04:45:22.643900+0000","last_became_peered":"2024-04-26T04:45:22.643900+0000","last_unstale":"2024-04-26T04:46:05.560161+0000","last_undegraded":"2024-04-26T04:46:05.560161+0000","last_fullsized":"2024-04-26T04:46:05.560161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:37.065136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.565456+0000","last_change":"2024-04-26T04:45:21.641364+0000","last_active":"2024-04-26T04:46:05.565456+0000","last_peered":"2024-04-26T04:46:05.565456+0000","last_clean":"2024-04-26T04:46:05.565456+0000","last_became_active":"2024-04-26T04:45:21.640716+0000","last_became_peered":"2024-04-26T04:45:21.640716+0000","last_unstale":"2024-04-26T04:46:05.565456+0000","last_undegraded":"2024-04-26T04:46:05.565456+0000","last_fullsized":"2024-04-26T04:46:05.565456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:51:16.247487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563011+0000","last_change":"2024-04-26T04:45:22.646855+0000","last_active":"2024-04-26T04:46:05.563011+0000","last_peered":"2024-04-26T04:46:05.563011+0000","last_clean":"2024-04-26T04:46:05.563011+0000","last_became_active":"2024-04-26T04:45:22.646549+0000","last_became_peered":"2024-04-26T04:45:22.646549+0000","last_unstale":"2024-04-26T04:46:05.563011+0000","last_undegraded":"2024-04-26T04:46:05.563011+0000","last_fullsized":"2024-04-26T04:46:05.563011+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:14:26.381684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560585+0000","last_change":"2024-04-26T04:45:21.633696+0000","last_active":"2024-04-26T04:46:05.560585+0000","last_peered":"2024-04-26T04:46:05.560585+0000","last_clean":"2024-04-26T04:46:05.560585+0000","last_became_active":"2024-04-26T04:45:21.633431+0000","last_became_peered":"2024-04-26T04:45:21.633431+0000","last_unstale":"2024-04-26T04:46:05.560585+0000","last_undegraded":"2024-04-26T04:46:05.560585+0000","last_fullsized":"2024-04-26T04:46:05.560585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:23:57.941279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562553+0000","last_change":"2024-04-26T04:45:22.641284+0000","last_active":"2024-04-26T04:46:05.562553+0000","last_peered":"2024-04-26T04:46:05.562553+0000","last_clean":"2024-04-26T04:46:05.562553+0000","last_became_active":"2024-04-26T04:45:22.640982+0000","last_became_peered":"2024-04-26T04:45:22.640982+0000","last_unstale":"2024-04-26T04:46:05.562553+0000","last_undegraded":"2024-04-26T04:46:05.562553+0000","last_fullsized":"2024-04-26T04:46:05.562553+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:26:16.595756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559539+0000","last_change":"2024-04-26T04:45:21.642451+0000","last_active":"2024-04-26T04:46:05.559539+0000","last_peered":"2024-04-26T04:46:05.559539+0000","last_clean":"2024-04-26T04:46:05.559539+0000","last_became_active":"2024-04-26T04:45:21.641428+0000","last_became_peered":"2024-04-26T04:45:21.641428+0000","last_unstale":"2024-04-26T04:46:05.559539+0000","last_undegraded":"2024-04-26T04:46:05.559539+0000","last_fullsized":"2024-04-26T04:46:05.559539+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:43:10.838400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563001+0000","last_change":"2024-04-26T04:45:22.642636+0000","last_active":"2024-04-26T04:46:05.563001+0000","last_peered":"2024-04-26T04:46:05.563001+0000","last_clean":"2024-04-26T04:46:05.563001+0000","last_became_active":"2024-04-26T04:45:22.642325+0000","last_became_peered":"2024-04-26T04:45:22.642325+0000","last_unstale":"2024-04-26T04:46:05.563001+0000","last_undegraded":"2024-04-26T04:46:05.563001+0000","last_fullsized":"2024-04-26T04:46:05.563001+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:01:24.302695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.562607+0000","last_change":"2024-04-26T04:45:21.643555+0000","last_active":"2024-04-26T04:46:05.562607+0000","last_peered":"2024-04-26T04:46:05.562607+0000","last_clean":"2024-04-26T04:46:05.562607+0000","last_became_active":"2024-04-26T04:45:21.643242+0000","last_became_peered":"2024-04-26T04:45:21.643242+0000","last_unstale":"2024-04-26T04:46:05.562607+0000","last_undegraded":"2024-04-26T04:46:05.562607+0000","last_fullsized":"2024-04-26T04:46:05.562607+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:07:39.491678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.559253+0000","last_change":"2024-04-26T04:45:22.650094+0000","last_active":"2024-04-26T04:46:05.559253+0000","last_peered":"2024-04-26T04:46:05.559253+0000","last_clean":"2024-04-26T04:46:05.559253+0000","last_became_active":"2024-04-26T04:45:22.649627+0000","last_became_peered":"2024-04-26T04:45:22.649627+0000","last_unstale":"2024-04-26T04:46:05.559253+0000","last_undegraded":"2024-04-26T04:46:05.559253+0000","last_fullsized":"2024-04-26T04:46:05.559253+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:39:30.883085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.560021+0000","last_change":"2024-04-26T04:45:21.632189+0000","last_active":"2024-04-26T04:46:05.560021+0000","last_peered":"2024-04-26T04:46:05.560021+0000","last_clean":"2024-04-26T04:46:05.560021+0000","last_became_active":"2024-04-26T04:45:21.631875+0000","last_became_peered":"2024-04-26T04:45:21.631875+0000","last_unstale":"2024-04-26T04:46:05.560021+0000","last_undegraded":"2024-04-26T04:46:05.560021+0000","last_fullsized":"2024-04-26T04:46:05.560021+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:02:56.126210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.563414+0000","last_change":"2024-04-26T04:45:22.648545+0000","last_active":"2024-04-26T04:46:05.563414+0000","last_peered":"2024-04-26T04:46:05.563414+0000","last_clean":"2024-04-26T04:46:05.563414+0000","last_became_active":"2024-04-26T04:45:22.648249+0000","last_became_peered":"2024-04-26T04:45:22.648249+0000","last_unstale":"2024-04-26T04:46:05.563414+0000","last_undegraded":"2024-04-26T04:46:05.563414+0000","last_fullsized":"2024-04-26T04:46:05.563414+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:17:05.175726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:46:05.916090+0000","last_change":"2024-04-26T04:45:21.629149+0000","last_active":"2024-04-26T04:46:05.916090+0000","last_peered":"2024-04-26T04:46:05.916090+0000","last_clean":"2024-04-26T04:46:05.916090+0000","last_became_active":"2024-04-26T04:45:21.628847+0000","last_became_peered":"2024-04-26T04:45:21.628847+0000","last_unstale":"2024-04-26T04:46:05.916090+0000","last_undegraded":"2024-04-26T04:46:05.916090+0000","last_fullsized":"2024-04-26T04:46:05.916090+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:03:05.754090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1316,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1347584,"data_stored":908605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.51900000000000002,"15min":0.52500000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.64300000000000002,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.504},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67800000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.60999999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.55600000000000005,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.66600000000000004,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.51800000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72499999999999998,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53700000000000003,"15min":0.53500000000000003},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.53000000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42699999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.69499999999999995,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.39100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70699999999999996,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.63400000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.621},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70399999999999996,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.39300000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.45700000000000002,"15min":0.45000000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.66600000000000004,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.436},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.52300000000000002,"15min":0.52500000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67600000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57399999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:45:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52500000000000002,"15min":0.52800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66100000000000003,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70199999999999996,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46600000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":907929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.52000000000000002,"15min":0.51800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.499,"5min":0.93300000000000005,"15min":1.121},"min":{"1min":0.38200000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67200000000000004,"5min":16.103999999999999,"15min":16.103999999999999},"last":0.504}]},{"osd":1,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.81100000000000005,"15min":0.91800000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70999999999999996,"5min":9.75,"15min":9.75},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.42499999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78100000000000003,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.75600000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52000000000000002,"15min":0.51900000000000002},"min":{"1min":0.38,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68000000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.76900000000000002,"15min":0.874},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.64700000000000002,"5min":9.7249999999999996,"15min":9.7249999999999996},"last":0.56100000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.44600000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.67100000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.95999999999999996,"15min":1.137},"min":{"1min":0.41999999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72199999999999998,"5min":16.128,"15min":16.128},"last":0.80300000000000005}]},{"osd":4,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.42999999999999999,"15min":0.40999999999999998},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70499999999999996,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.871}]},{"osd":5,"last update":"Fri Apr 26 04:46:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.44700000000000001,"15min":0.434},"min":{"1min":0.34999999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68500000000000005,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.495,"15min":0.48099999999999998},"min":{"1min":0.36899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.84599999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:45:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.51700000000000002,"15min":0.50700000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.47199999999999998,"15min":0.46200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.83699999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":316204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:46:19 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.38,"15min":0.33200000000000002},"min":{"1min":0.26700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.59699999999999998,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.34999999999999998},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.35599999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:45:42 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38100000000000001},"min":{"1min":0.26800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.307}]},{"osd":3,"last update":"Fri Apr 26 04:46:04 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.48999999999999999,"15min":0.505},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61199999999999999,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.44500000000000001,"15min":0.46500000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56200000000000006,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.40899999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:45:24 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.58699999999999997,"15min":0.58099999999999996},"min":{"1min":0.47199999999999998,"5min":0.376,"15min":0.376},"max":{"1min":0.81699999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.73099999999999998,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.505}]},{"osd":5,"last update":"Fri Apr 26 04:45:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.495,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76200000000000001,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58699999999999997,"15min":0.58499999999999996},"min":{"1min":0.52800000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.83899999999999997,"5min":0.874,"15min":0.874},"last":0.55400000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:46:19 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.61099999999999999,"15min":0.60899999999999999},"min":{"1min":0.46700000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.60099999999999998,"15min":0.59299999999999997},"min":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.626}]},{"osd":7,"last update":"Fri Apr 26 04:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.64000000000000001,"15min":0.64200000000000002},"min":{"1min":0.48199999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.75600000000000001,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.532,"5min":0.83199999999999996,"15min":0.71599999999999997},"min":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":16.957999999999998,"5min":16.957999999999998,"15min":16.957999999999998},"last":0.65200000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738440,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.376,"15min":0.35299999999999998},"min":{"1min":0.23000000000000001,"5min":0.192,"15min":0.192},"max":{"1min":0.46999999999999997,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.38800000000000001,"15min":0.34699999999999998},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.379}]},{"osd":2,"last update":"Fri Apr 26 04:45:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.40799999999999997,"15min":0.41099999999999998},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.60299999999999998,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.40899999999999997,"15min":0.39800000000000002},"min":{"1min":0.27600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.59099999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.27700000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.48499999999999999,"15min":0.49299999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.60999999999999999,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.48299999999999998,"15min":0.504},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.57899999999999996,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.36599999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:45:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58499999999999996,"15min":0.58199999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75900000000000001,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71999999999999997,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.496}]},{"osd":5,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.622,"15min":0.623},"min":{"1min":0.41099999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.876,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64400000000000002,"15min":0.64600000000000002},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.94599999999999995,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.50800000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:46:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.61599999999999999,"15min":0.61899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.65400000000000003,"15min":0.66900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.80800000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53200000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:45:37 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.46899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.79400000000000004,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85699999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:43 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.68600000000000005,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.35299999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:45:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.755,"5min":0.875,"15min":0.875},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.437,"15min":0.439},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67200000000000004,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.377}]},{"osd":3,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48699999999999999,"15min":0.47699999999999998},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.47899999999999998,"15min":0.48599999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.30199999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:45:24 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79600000000000004,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.625},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.59099999999999997,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83099999999999996,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.44400000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:45:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63,"15min":0.64400000000000002},"min":{"1min":0.45800000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.41199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.86499999999999999,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.433}]},{"osd":6,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.64600000000000002,"15min":0.65000000000000002},"min":{"1min":0.46600000000000003,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.46999999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:45:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.626,"15min":0.63100000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.70399999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:46:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.42699999999999999,"15min":0.40300000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.625,"5min":3.625,"15min":3.625},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.46500000000000002,"15min":0.44700000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.827,"5min":3.827,"15min":3.827},"last":0.39700000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:46:02 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.51800000000000002,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2000000000000002,"5min":7.2000000000000002,"15min":7.2000000000000002},"last":0.435},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.52200000000000002,"15min":0.47599999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.1289999999999996,"5min":9.1289999999999996,"15min":9.1289999999999996},"last":0.29599999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:46:02 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.55300000000000005,"15min":0.51000000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.7759999999999998,"5min":8.7759999999999998,"15min":8.7759999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.495,"15min":0.47299999999999998},"min":{"1min":0.27700000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.581,"5min":3.581,"15min":3.581},"last":0.34300000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:45:22 2024","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.61399999999999999,"15min":0.57999999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":3.6720000000000002,"5min":3.6720000000000002,"15min":3.6720000000000002},"last":0.625},{"interface":"front","average":{"1min":1.133,"5min":0.70299999999999996,"15min":0.63100000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":8.7110000000000003,"5min":8.7110000000000003,"15min":8.7110000000000003},"last":0.54900000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:45:51 2024","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.65000000000000002,"15min":0.624},"min":{"1min":0.49099999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.4079999999999999,"5min":3.4079999999999999,"15min":3.4079999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.71399999999999997,"15min":0.65000000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.7449999999999992,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.61599999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.72299999999999998,"15min":0.67300000000000004},"min":{"1min":0.45900000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.6660000000000004,"5min":8.6660000000000004,"15min":8.6660000000000004},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.69099999999999995,"15min":0.66700000000000004},"min":{"1min":0.46400000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.60399999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.70399999999999996,"15min":0.67900000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":3.4820000000000002,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.67700000000000005,"15min":0.65100000000000002},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":3.722,"5min":3.722,"15min":3.722},"last":0.67100000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298088,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447064,"statfs":{"total":95995035648,"available":95689793536,"internally_reserved":0,"allocated":745472,"data_stored":317902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.52800000000000002,"15min":0.51600000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54000000000000004},"min":{"1min":0.40400000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.79200000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.59999999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52500000000000002,"15min":0.51600000000000001},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53700000000000003,"15min":0.53400000000000003},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73699999999999999,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.55000000000000004,"15min":0.54500000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83299999999999996,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.67600000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:45:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54300000000000004,"15min":0.53100000000000003},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78600000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.56299999999999994,"15min":0.55500000000000005},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.68899999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:45:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.41699999999999998,"15min":0.38700000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.64900000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.627},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.42899999999999999,"15min":0.39400000000000002},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.70799999999999996,"5min":0.88,"15min":0.88},"last":0.47599999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:46:13 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.44400000000000001,"15min":0.41899999999999998},"min":{"1min":0.29799999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.745,"5min":0.876,"15min":0.876},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44600000000000001,"15min":0.41799999999999998},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.66200000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51400000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:45:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52200000000000002,"15min":0.51300000000000001},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52100000000000002,"15min":0.51600000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53600000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54400000000000004,"15min":0.53900000000000003},"min":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.70099999999999996,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69699999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.61899999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.61699999999999999,"15min":0.56499999999999995},"min":{"1min":0.443,"5min":0.374,"15min":0.374},"max":{"1min":0.71999999999999997,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.629,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55100000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.53100000000000003,"15min":0.52300000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65300000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.61099999999999999,"15min":0.57899999999999996},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.5709999999999997,"5min":4.5709999999999997,"15min":4.5709999999999997},"last":0.68899999999999995}]},{"osd":3,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.63100000000000001,"15min":0.57699999999999996},"min":{"1min":0.42299999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71399999999999997,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56200000000000006,"15min":0.55000000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73899999999999999,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.57099999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:45:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.442,"15min":0.436},"min":{"1min":0.32100000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.46200000000000002,"15min":0.44700000000000001},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.47199999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:46:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.49199999999999999,"15min":0.47699999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.505,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.308,"15min":0.308},"max":{"1min":0.76000000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.51600000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:45:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.51700000000000002,"15min":0.51200000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.434},{"interface":"front","average":{"1min":0.505,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.35299999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.45500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:46:25.134 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:46:25.134 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:46:25.134 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:46:25.134 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:46:25.406 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:25.779 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:25 smithi005 ceph-mon[26371]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:46:25.780 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:25 smithi005 ceph-mon[36961]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:46:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:25 smithi111 ceph-mon[30255]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:46:26.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:26 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:26] "GET /metrics HTTP/1.1" 200 239231 "" "Prometheus/2.43.0" 2024-04-26T04:46:26.907 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:26 smithi005 ceph-mon[36961]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:26.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:26 smithi005 ceph-mon[26371]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:26 smithi111 ceph-mon[30255]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:27.354 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T04:46:27.355 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:46:27.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:27 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4119230225' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:46:27.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:27 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4119230225' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:46:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:27 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4119230225' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:46:28.057 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:46:28.057 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:46:28.067 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-26T04:46:28.067 INFO:teuthology.run_tasks:Running task parallel... 2024-04-26T04:46:28.132 INFO:teuthology.task.parallel:starting parallel... 2024-04-26T04:46:28.132 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:46:28.132 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-26T04:46:28.136 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-04-26T04:46:28.137 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-26T04:46:28.137 DEBUG:teuthology.orchestra.run.smithi005:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:46:28.137 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:46:28.138 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T04:46:28.138 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi005.front.sepia.ceph.com 2024-04-26T04:46:28.138 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-26T04:46:28.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:46:28.157 INFO:teuthology.orchestra.run.smithi005.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-04-26T04:46:28.157 DEBUG:teuthology.orchestra.run.smithi005:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:46:28.212 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-04-26T04:46:28.213 DEBUG:teuthology.orchestra.run.smithi005:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-04-26T04:46:28.269 INFO:tasks.workunit:timeout=3h 2024-04-26T04:46:28.269 INFO:tasks.workunit:cleanup=True 2024-04-26T04:46:28.269 DEBUG:teuthology.orchestra.run.smithi005:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:46:28.325 INFO:tasks.workunit.client.0.smithi005.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-26T04:46:28.419 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:28.840 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:28 smithi005 ceph-mon[26371]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:28.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:28 smithi005 ceph-mon[36961]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:28 smithi111 ceph-mon[30255]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:30.634 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:46:30] ENGINE Bus STOPPING 2024-04-26T04:46:30.634 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:46:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:46:30.635 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:46:30] ENGINE Bus STOPPED 2024-04-26T04:46:30.635 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:46:30] ENGINE Bus STARTING 2024-04-26T04:46:30.635 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:46:30] ENGINE Serving on http://:::9283 2024-04-26T04:46:30.635 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:46:30] ENGINE Bus STARTED 2024-04-26T04:46:30.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-mon[36961]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:30.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/520999587' entity='client.admin' 2024-04-26T04:46:30.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:30.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-mon[26371]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:30.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/520999587' entity='client.admin' 2024-04-26T04:46:30.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:30 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:30 smithi111 ceph-mon[30255]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:30 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/520999587' entity='client.admin' 2024-04-26T04:46:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:30 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:31.043 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T04:46:31.345 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:31.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:31 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:31.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:31 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:31.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:31 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:31.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:31 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:31.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:31 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:31.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:31 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:32.001 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:31 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:32.001 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:31 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:32.001 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:31 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:32.002 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:46:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:46:32.901 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:32 smithi005 ceph-mon[26371]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:32.902 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:32 smithi005 ceph-mon[36961]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:32 smithi111 ceph-mon[30255]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:34.267 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T04:46:34.536 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:34.966 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:34 smithi005 ceph-mon[26371]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:34.966 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:34 smithi005 ceph-mon[36961]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:34 smithi111 ceph-mon[30255]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:35.916 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:35 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:35.916 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:35 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:36.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:35 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:36.169 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:36] "GET /metrics HTTP/1.1" 200 239219 "" "Prometheus/2.43.0" 2024-04-26T04:46:36.499 INFO:teuthology.orchestra.run.smithi005.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:36.949 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-mon[26371]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:36.949 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:36.949 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:36.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-mon[36961]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:36.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:36.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:36 smithi111 ceph-mon[30255]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:37.296 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:46:37.574 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:37.646 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[26371]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:37.646 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[26371]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:37.646 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:37.646 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:37.646 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:37.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[36961]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:37.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[36961]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:37.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:37.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:37.897 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:37 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:37 smithi111 ceph-mon[30255]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:37 smithi111 ceph-mon[30255]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:37 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:37 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:37 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:46:38.829 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:38 smithi005 ceph-mon[26371]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:38.829 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:38 smithi005 ceph-mon[26371]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:38.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:38 smithi005 ceph-mon[36961]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:38.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:38 smithi005 ceph-mon[36961]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:38 smithi111 ceph-mon[30255]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:38 smithi111 ceph-mon[30255]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:46:39.699 INFO:teuthology.orchestra.run.smithi005.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:alertmanager.a smithi005 *:9093,9094 running (83s) 19s ago 2m 23.7M - 0.25.0 c8568f914cd2 8b891aba4f92 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:grafana.a smithi111 *:3000 running (2m) 39s ago 2m 82.3M - 9.4.7 954c08fa6188 d26b90f67e22 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi005.zdnlsb smithi005 running (64s) 19s ago 64s 15.9M - 17.2.7-902-gc61472c8 0de407c1b3af 7ddb93a6b5a1 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi111.furhfd smithi111 running (62s) 39s ago 61s 14.5M - 17.2.7-902-gc61472c8 0de407c1b3af 3e206ca72f2e 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.x smithi111 *:8443 running (6m) 39s ago 6m 432M - 17.2.7-902-gc61472c8 0de407c1b3af 587d3c982d8c 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.y smithi005 *:9283 running (8m) 19s ago 8m 476M - 17.2.7-902-gc61472c8 0de407c1b3af 4e6b89d741aa 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mon.a smithi005 running (8m) 19s ago 8m 47.1M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 6b2f87797829 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mon.b smithi111 running (6m) 39s ago 6m 35.9M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 20923c1c0e03 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:mon.c smithi005 running (6m) 19s ago 6m 37.4M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 78e8c297f4e2 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.a smithi005 *:9100 running (116s) 19s ago 115s 13.9M - 1.5.0 0da6a335fe13 12c05c1a199c 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.b smithi111 *:9100 running (112s) 39s ago 112s 8242k - 1.5.0 0da6a335fe13 2e3c3a24be0d 2024-04-26T04:46:39.700 INFO:teuthology.orchestra.run.smithi005.stdout:osd.0 smithi005 running (6m) 19s ago 6m 42.7M 4017M 17.2.7-902-gc61472c8 0de407c1b3af b99cc094c6d7 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.1 smithi005 running (5m) 19s ago 5m 44.0M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 9e578b598142 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.2 smithi005 running (5m) 19s ago 5m 41.0M 4017M 17.2.7-902-gc61472c8 0de407c1b3af f25e1cd874ca 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.3 smithi005 running (4m) 19s ago 4m 40.7M 4017M 17.2.7-902-gc61472c8 0de407c1b3af fd23fb067fc9 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.4 smithi111 running (4m) 39s ago 4m 41.2M 4016M 17.2.7-902-gc61472c8 0de407c1b3af e2994c2a7aa2 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.5 smithi111 running (4m) 39s ago 4m 41.7M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 5da203330767 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.6 smithi111 running (3m) 39s ago 3m 40.0M 4016M 17.2.7-902-gc61472c8 0de407c1b3af b7798f2bbff0 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:osd.7 smithi111 running (3m) 39s ago 3m 39.9M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 60f1bafb36a6 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi005.stdout:prometheus.a smithi111 *:9095 running (77s) 39s ago 2m 40.1M - 2.43.0 a07b618ecd1d f0899160bb49 2024-04-26T04:46:40.405 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-26T04:46:40.686 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:40.906 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:40 smithi005 ceph-mon[36961]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:40.907 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:40 smithi005 ceph-mon[26371]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:40 smithi111 ceph-mon[30255]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:41 smithi111 ceph-mon[30255]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:41.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:46:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:46:41.922 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:41 smithi005 ceph-mon[36961]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:41.922 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:41 smithi005 ceph-mon[26371]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:42.231 DEBUG:teuthology.orchestra.run.smithi005:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-26T04:46:42.277 INFO:tasks.workunit.client.0.smithi005.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-26T04:46:42.279 INFO:tasks.workunit.client.0.smithi005.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:46:42.279 INFO:tasks.workunit.client.0.smithi005.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-26T04:46:42.841 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:42 smithi005 ceph-mon[26371]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:42.841 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:42 smithi005 ceph-mon[36961]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:42.965 INFO:teuthology.orchestra.run.smithi005.stdout:wait for minority of mons to upgrade 2024-04-26T04:46:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:42 smithi111 ceph-mon[30255]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:43.997 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-26T04:46:44.045 INFO:tasks.workunit.client.0.smithi005.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-26T04:46:44.107 INFO:tasks.workunit.client.0.smithi005.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-26T04:46:44.156 INFO:tasks.workunit.client.0.smithi005.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:46:44.157 INFO:tasks.workunit.client.0.smithi005.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:46:44.157 INFO:tasks.workunit.client.0.smithi005.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-26T04:46:44.199 INFO:tasks.workunit.client.0.smithi005.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:46:44.203 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-26T04:46:44.203 DEBUG:teuthology.orchestra.run.smithi005:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-26T04:46:44.223 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-04-26T04:46:44.225 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-04-26T04:46:44.226 DEBUG:teuthology.orchestra.run.smithi005:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-04-26T04:46:44.308 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:44.315 INFO:tasks.workunit.client.0.smithi005.stderr:warning: failed to enable lockdep 2024-04-26T04:46:44.315 INFO:tasks.workunit.client.0.smithi005.stdout:seed 77850 2024-04-26T04:46:44.315 INFO:tasks.workunit.client.0.smithi005.stdout:[==========] Running 415 tests from 16 test suites. 2024-04-26T04:46:44.315 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] Global test environment set-up. 2024-04-26T04:46:44.315 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 1 test from TestJournalStress 2024-04-26T04:46:44.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:44 smithi005 ceph-mon[36961]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:44.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:44 smithi005 ceph-mon[26371]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:44 smithi111 ceph-mon[30255]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:45.901 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[26371]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:46:45.906 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-04-26T04:46:46.201 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/592152008' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:46.201 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:46.201 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[26371]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:46.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[36961]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:46:46.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/592152008' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:46.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:46.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:45 smithi005 ceph-mon[36961]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:46.202 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:46] "GET /metrics HTTP/1.1" 200 239219 "" "Prometheus/2.43.0" 2024-04-26T04:46:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:45 smithi111 ceph-mon[30255]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:46:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:45 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/592152008' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:45 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:45 smithi111 ceph-mon[30255]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:46:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[26371]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:46:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2286295937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:46:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[36961]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:46:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:46 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2286295937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:46 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:46:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:46 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:46 smithi111 ceph-mon[30255]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:46:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:46 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2286295937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:47.930 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:47.930 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalStress.DiscardWithPruneWriteOverlap (2024 ms) 2024-04-26T04:46:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:47 smithi005 ceph-mon[26371]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:46:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:47 smithi005 ceph-mon[26371]: pgmap v98: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:47 smithi005 ceph-mon[36961]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:46:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:47 smithi005 ceph-mon[36961]: pgmap v98: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:47 smithi111 ceph-mon[30255]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:46:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:47 smithi111 ceph-mon[30255]: pgmap v98: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:48.926 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 1 test from TestJournalStress (2024 ms total) 2024-04-26T04:46:48.926 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:46:48.926 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 16 tests from TestJournalReplay 2024-04-26T04:46:49.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:48 smithi005 ceph-mon[36961]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:46:49.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:48 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:46:49.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:48 smithi005 ceph-mon[36961]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:46:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:48 smithi005 ceph-mon[26371]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:46:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:48 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:46:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:48 smithi005 ceph-mon[26371]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:46:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:48 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:46:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:48 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:46:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:48 smithi111 ceph-mon[30255]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:46:50.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:49 smithi005 ceph-mon[26371]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:46:50.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:49 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/386496878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:50.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:49 smithi005 ceph-mon[26371]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:49 smithi005 ceph-mon[36961]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:46:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:49 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/386496878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:49 smithi005 ceph-mon[36961]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:49 smithi111 ceph-mon[30255]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:46:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:49 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/386496878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:49 smithi111 ceph-mon[30255]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:50.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-04-26T04:46:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:50 smithi005 ceph-mon[26371]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:46:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:50 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1103700703' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:50 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:50 smithi005 ceph-mon[36961]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:46:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:50 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1103700703' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:50 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:50 smithi111 ceph-mon[30255]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:46:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:50 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1103700703' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:50 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:51.931 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:46:51 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:46:52.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1103700703' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:52.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[26371]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:46:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[26371]: pgmap v104: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/472509396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1103700703' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[36961]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:46:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[36961]: pgmap v104: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:51 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/472509396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:51 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1103700703' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:51 smithi111 ceph-mon[30255]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:46:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:51 smithi111 ceph-mon[30255]: pgmap v104: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:51 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/472509396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:52.951 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:52.951 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2020 ms) 2024-04-26T04:46:52.951 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-04-26T04:46:52.960 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:52.960 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.AioWriteEvent (9 ms) 2024-04-26T04:46:52.960 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-04-26T04:46:52.967 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:52.967 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.AioFlushEvent (7 ms) 2024-04-26T04:46:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-04-26T04:46:52.978 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:52.978 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.SnapCreate (11 ms) 2024-04-26T04:46:52.978 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-04-26T04:46:52.986 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:52.986 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.SnapProtect (8 ms) 2024-04-26T04:46:52.986 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-04-26T04:46:52.993 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:52.993 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.SnapUnprotect (7 ms) 2024-04-26T04:46:52.993 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-04-26T04:46:53.001 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.001 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.SnapRename (8 ms) 2024-04-26T04:46:53.001 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-04-26T04:46:53.008 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.008 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.SnapRollback (7 ms) 2024-04-26T04:46:53.008 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-04-26T04:46:53.016 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.017 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.SnapRemove (9 ms) 2024-04-26T04:46:53.017 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.Rename 2024-04-26T04:46:53.024 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.024 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.Rename (7 ms) 2024-04-26T04:46:53.024 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.Resize 2024-04-26T04:46:53.032 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.033 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.Resize (8 ms) 2024-04-26T04:46:53.033 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.Flatten 2024-04-26T04:46:53.040 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.040 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.Flatten (8 ms) 2024-04-26T04:46:53.040 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-04-26T04:46:53.047 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.048 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.UpdateFeatures (8 ms) 2024-04-26T04:46:53.048 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-04-26T04:46:53.056 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.056 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.MetadataSet (8 ms) 2024-04-26T04:46:53.056 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-04-26T04:46:53.064 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.064 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.MetadataRemove (8 ms) 2024-04-26T04:46:53.064 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-04-26T04:46:53.072 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:53.072 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalReplay.ObjectPosition (8 ms) 2024-04-26T04:46:53.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:52 smithi005 ceph-mon[26371]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:46:53.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:52 smithi005 ceph-mon[36961]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:46:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:52 smithi111 ceph-mon[30255]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:46:53.952 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 16 tests from TestJournalReplay (2141 ms total) 2024-04-26T04:46:53.952 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:46:53.952 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 4 tests from TestJournalEntries 2024-04-26T04:46:54.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:53 smithi005 ceph-mon[36961]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:46:54.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:53 smithi005 ceph-mon[36961]: pgmap v107: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:53 smithi005 ceph-mon[26371]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:46:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:53 smithi005 ceph-mon[26371]: pgmap v107: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:53 smithi111 ceph-mon[30255]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:46:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:53 smithi111 ceph-mon[30255]: pgmap v107: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:55.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:54 smithi005 ceph-mon[26371]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:46:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:54 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3099127761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:54 smithi005 ceph-mon[36961]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:46:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:54 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3099127761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:54 smithi111 ceph-mon[30255]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:46:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:54 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3099127761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:55.957 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-04-26T04:46:56.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[36961]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:46:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2931736972' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[36961]: pgmap v110: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[26371]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:46:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2931736972' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:56.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:55 smithi005 ceph-mon[26371]: pgmap v110: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:56.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:46:56] "GET /metrics HTTP/1.1" 200 241543 "" "Prometheus/2.43.0" 2024-04-26T04:46:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:55 smithi111 ceph-mon[30255]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:46:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:55 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2931736972' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:55 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:46:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:55 smithi111 ceph-mon[30255]: pgmap v110: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:57.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[36961]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4138613155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[26371]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:46:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:56 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4138613155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:56 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:46:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:56 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:46:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:56 smithi111 ceph-mon[30255]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:46:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:56 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4138613155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:57.976 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:57.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalEntries.AioWrite (2019 ms) 2024-04-26T04:46:57.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-04-26T04:46:57.988 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:57.988 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalEntries.AioDiscard (11 ms) 2024-04-26T04:46:57.988 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalEntries.AioDiscardWithPrune 2024-04-26T04:46:57.995 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:57.995 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalEntries.AioDiscardWithPrune (8 ms) 2024-04-26T04:46:57.996 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-04-26T04:46:58.003 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:46:58.004 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestJournalEntries.AioFlush (9 ms) 2024-04-26T04:46:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:57 smithi005 ceph-mon[26371]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:46:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:57 smithi005 ceph-mon[26371]: pgmap v113: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:57 smithi005 ceph-mon[36961]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:46:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:57 smithi005 ceph-mon[36961]: pgmap v113: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:57 smithi111 ceph-mon[30255]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:46:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:57 smithi111 ceph-mon[30255]: pgmap v113: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:58.971 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 4 tests from TestJournalEntries (2047 ms total) 2024-04-26T04:46:58.971 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:46:58.971 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 2 tests from TestTrash 2024-04-26T04:46:59.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:58 smithi005 ceph-mon[26371]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:46:59.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:58 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:46:59.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:58 smithi005 ceph-mon[26371]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:46:59.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:58 smithi005 ceph-mon[36961]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:46:59.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:58 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:46:59.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:58 smithi005 ceph-mon[36961]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:46:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:58 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:46:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:58 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:46:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:58 smithi111 ceph-mon[30255]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:47:00.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:59 smithi005 ceph-mon[36961]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:47:00.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:59 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2081356216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:00.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:46:59 smithi005 ceph-mon[36961]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:00.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:59 smithi005 ceph-mon[26371]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:47:00.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:59 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2081356216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:00.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:46:59 smithi005 ceph-mon[26371]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:59 smithi111 ceph-mon[30255]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:47:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:59 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2081356216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:46:59 smithi111 ceph-mon[30255]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:00.983 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-04-26T04:47:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:00 smithi005 ceph-mon[26371]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:47:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:00 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3359276291' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:00 smithi005 ceph-mon[36961]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:47:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:00 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3359276291' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:00 smithi111 ceph-mon[30255]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:47:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:00 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3359276291' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:01.984 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:47:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:01 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3359276291' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:01 smithi111 ceph-mon[30255]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:47:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:01 smithi111 ceph-mon[30255]: pgmap v119: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:01 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1258188600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3359276291' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[26371]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:47:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[26371]: pgmap v119: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:02.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1258188600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3359276291' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[36961]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:47:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[36961]: pgmap v119: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:01 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1258188600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:03.079 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestTrash.UserRemovingSource (2096 ms) 2024-04-26T04:47:03.079 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-04-26T04:47:03.124 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestTrash.RestoreMirroringSource (44 ms) 2024-04-26T04:47:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:02 smithi111 ceph-mon[30255]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T04:47:03.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:02 smithi005 ceph-mon[26371]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T04:47:03.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:02 smithi005 ceph-mon[36961]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T04:47:04.004 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 2 tests from TestTrash (2140 ms total) 2024-04-26T04:47:04.004 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:47:04.004 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 1 test from TestOperations 2024-04-26T04:47:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:03 smithi111 ceph-mon[30255]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T04:47:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:03 smithi111 ceph-mon[30255]: pgmap v122: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:04.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:03 smithi005 ceph-mon[26371]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T04:47:04.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:03 smithi005 ceph-mon[26371]: pgmap v122: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:04.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:03 smithi005 ceph-mon[36961]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T04:47:04.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:03 smithi005 ceph-mon[36961]: pgmap v122: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:04 smithi111 ceph-mon[30255]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T04:47:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:04 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1315154954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:05.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:04 smithi005 ceph-mon[36961]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T04:47:05.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:04 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1315154954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:04 smithi005 ceph-mon[26371]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T04:47:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:04 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1315154954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:06.012 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-04-26T04:47:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:06 smithi111 ceph-mon[30255]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:47:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:06 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2481607463' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:06 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:06 smithi111 ceph-mon[30255]: pgmap v125: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:06.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:06] "GET /metrics HTTP/1.1" 200 242073 "" "Prometheus/2.43.0" 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[26371]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2481607463' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[26371]: pgmap v125: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[36961]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2481607463' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[36961]: pgmap v125: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:07 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:07 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:07 smithi111 ceph-mon[30255]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:47:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:07 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3925941473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:07.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:07.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:07.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[26371]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:47:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3925941473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:07.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:07.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:07.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[36961]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:47:07.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:07 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3925941473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:08.032 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:47:08.032 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2020 ms) 2024-04-26T04:47:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:08 smithi111 ceph-mon[30255]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T04:47:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:08 smithi111 ceph-mon[30255]: pgmap v128: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:08.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:08 smithi005 ceph-mon[26371]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T04:47:08.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:08 smithi005 ceph-mon[26371]: pgmap v128: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:08 smithi005 ceph-mon[36961]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T04:47:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:08 smithi005 ceph-mon[36961]: pgmap v128: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:09.029 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 1 test from TestOperations (2020 ms total) 2024-04-26T04:47:09.029 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:47:09.029 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 4 tests from TestObjectMap 2024-04-26T04:47:09.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[36961]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:47:09.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:47:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[36961]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T04:47:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3002715855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[26371]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:47:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:47:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[26371]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T04:47:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:09 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3002715855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:09 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:47:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:09 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:47:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:09 smithi111 ceph-mon[30255]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T04:47:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:09 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3002715855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:10.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[36961]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[36961]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[26371]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[26371]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:10 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:10 smithi111 ceph-mon[30255]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T04:47:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:10 smithi111 ceph-mon[30255]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:10 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:10 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:10 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:11.045 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-04-26T04:47:11.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi111 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3300629134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:47:11.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:47:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi111 2024-04-26T04:47:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:47:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3300629134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:11 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi111 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3300629134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:12.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:47:12.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:12.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[36961]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:47:12.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3286871131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:12.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[36961]: pgmap v134: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:12.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:12.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[26371]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:47:12.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3286871131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:12.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:12 smithi005 ceph-mon[26371]: pgmap v134: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:12 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:12 smithi111 ceph-mon[30255]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:47:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:12 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3286871131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:12 smithi111 ceph-mon[30255]: pgmap v134: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:13.074 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2029 ms) 2024-04-26T04:47:13.074 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-04-26T04:47:13.100 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (26 ms) 2024-04-26T04:47:13.100 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-04-26T04:47:13.134 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (34 ms) 2024-04-26T04:47:13.134 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-04-26T04:47:13.158 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (24 ms) 2024-04-26T04:47:13.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:13 smithi005 ceph-mon[36961]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T04:47:13.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:13 smithi005 ceph-mon[26371]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T04:47:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:13 smithi111 ceph-mon[30255]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T04:47:14.061 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 4 tests from TestObjectMap (2113 ms total) 2024-04-26T04:47:14.061 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:47:14.061 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 2 tests from TestMirroringWatcher 2024-04-26T04:47:14.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:14 smithi005 ceph-mon[26371]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T04:47:14.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:14 smithi005 ceph-mon[26371]: pgmap v137: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:14.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:14 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/766362600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:14 smithi005 ceph-mon[36961]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T04:47:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:14 smithi005 ceph-mon[36961]: pgmap v137: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:14 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/766362600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:14 smithi111 ceph-mon[30255]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T04:47:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:14 smithi111 ceph-mon[30255]: pgmap v137: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:14 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/766362600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:15.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:15 smithi005 ceph-mon[26371]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T04:47:15.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:15 smithi005 ceph-mon[36961]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T04:47:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:15 smithi111 ceph-mon[30255]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T04:47:16.069 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-04-26T04:47:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[26371]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:47:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2639835157' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[26371]: pgmap v140: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:16.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:16] "GET /metrics HTTP/1.1" 200 242073 "" "Prometheus/2.43.0" 2024-04-26T04:47:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[36961]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:47:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2639835157' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:16.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:16 smithi005 ceph-mon[36961]: pgmap v140: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:16 smithi111 ceph-mon[30255]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:47:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:16 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2639835157' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:16 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:16 smithi111 ceph-mon[30255]: pgmap v140: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:17.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[26371]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[26371]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3961861350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[36961]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[36961]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:47:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:17 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3961861350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:17 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:17 smithi111 ceph-mon[30255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:17 smithi111 ceph-mon[30255]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:47:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:17 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3961861350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:18.108 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2039 ms) 2024-04-26T04:47:18.109 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-04-26T04:47:18.130 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (22 ms) 2024-04-26T04:47:18.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:18 smithi005 ceph-mon[26371]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T04:47:18.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:18 smithi005 ceph-mon[26371]: pgmap v143: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:18 smithi005 ceph-mon[36961]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T04:47:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:18 smithi005 ceph-mon[36961]: pgmap v143: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:18 smithi111 ceph-mon[30255]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T04:47:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:18 smithi111 ceph-mon[30255]: pgmap v143: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:19.087 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 2 tests from TestMirroringWatcher (2061 ms total) 2024-04-26T04:47:19.087 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:47:19.087 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 57 tests from TestMigration 2024-04-26T04:47:19.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[36961]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:47:19.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[36961]: Cluster is now healthy 2024-04-26T04:47:19.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[36961]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T04:47:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/775484246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[26371]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:47:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[26371]: Cluster is now healthy 2024-04-26T04:47:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[26371]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T04:47:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:19 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/775484246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:19 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:47:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:19 smithi111 ceph-mon[30255]: Cluster is now healthy 2024-04-26T04:47:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:19 smithi111 ceph-mon[30255]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T04:47:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:19 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/775484246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:20 smithi005 ceph-mon[26371]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T04:47:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:20 smithi005 ceph-mon[26371]: pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:20 smithi005 ceph-mon[36961]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T04:47:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:20 smithi005 ceph-mon[36961]: pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:20 smithi111 ceph-mon[30255]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T04:47:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:20 smithi111 ceph-mon[30255]: pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[26371]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T04:47:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2824228063' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3950062606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[36961]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T04:47:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2824228063' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:21 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3950062606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:22.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:47:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:21 smithi111 ceph-mon[30255]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T04:47:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:21 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2824228063' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:47:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:21 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:21 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3950062606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:22.096 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.Empty 2024-04-26T04:47:22.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2824228063' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:22.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[26371]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T04:47:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[26371]: pgmap v149: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[26371]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T04:47:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2824228063' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[36961]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T04:47:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[36961]: pgmap v149: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:22 smithi005 ceph-mon[36961]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T04:47:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:22 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2824228063' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:47:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:22 smithi111 ceph-mon[30255]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T04:47:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:22 smithi111 ceph-mon[30255]: pgmap v149: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:22 smithi111 ceph-mon[30255]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T04:47:24.137 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:24.147 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:24.288 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:24.351 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:24.431 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.Empty (2335 ms) 2024-04-26T04:47:24.431 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.OtherName 2024-04-26T04:47:24.460 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:24.470 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:24 smithi005 ceph-mon[36961]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T04:47:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:24 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3537593692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:24 smithi005 ceph-mon[36961]: pgmap v152: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:24 smithi005 ceph-mon[26371]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T04:47:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:24 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3537593692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:24 smithi005 ceph-mon[26371]: pgmap v152: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:24 smithi111 ceph-mon[30255]: osdmap e110: 8 total, 8 up, 8 in 2024-04-26T04:47:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:24 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3537593692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:24 smithi111 ceph-mon[30255]: pgmap v152: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:24.591 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:24.651 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:24.729 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.OtherName (298 ms) 2024-04-26T04:47:24.729 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.OtherPool 2024-04-26T04:47:24.757 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:24.767 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:25.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:25 smithi005 ceph-mon[26371]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:25.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:25 smithi005 ceph-mon[26371]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T04:47:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:25 smithi005 ceph-mon[36961]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:25 smithi005 ceph-mon[36961]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T04:47:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:25 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:47:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:25 smithi111 ceph-mon[30255]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T04:47:26.211 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:26.275 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:26.356 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.OtherPool (1627 ms) 2024-04-26T04:47:26.356 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.OtherNamespace 2024-04-26T04:47:26.384 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:26.394 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:26.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:26] "GET /metrics HTTP/1.1" 200 242444 "" "Prometheus/2.43.0" 2024-04-26T04:47:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-mon[26371]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T04:47:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-mon[26371]: pgmap v155: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1539978170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-mon[36961]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T04:47:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-mon[36961]: pgmap v155: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:26 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1539978170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:26 smithi111 ceph-mon[30255]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T04:47:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:26 smithi111 ceph-mon[30255]: pgmap v155: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:26 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1539978170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:26.518 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:26.580 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:26.656 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.OtherNamespace (300 ms) 2024-04-26T04:47:26.656 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.DataPool 2024-04-26T04:47:26.682 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:26.692 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:26.825 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:26.892 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:26.968 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.DataPool (312 ms) 2024-04-26T04:47:26.968 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-04-26T04:47:26.997 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:27.006 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:27.136 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:47:27.226 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.AbortAfterPrepare (258 ms) 2024-04-26T04:47:27.227 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-04-26T04:47:27.254 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:27.263 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:27.357 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (130 ms) 2024-04-26T04:47:27.358 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-04-26T04:47:27.385 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:27.394 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:27.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:27 smithi005 ceph-mon[26371]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T04:47:27.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:27 smithi005 ceph-mon[36961]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T04:47:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:27 smithi111 ceph-mon[30255]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T04:47:27.525 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:27.593 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:47:27.687 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.AbortAfterExecute (330 ms) 2024-04-26T04:47:27.687 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-04-26T04:47:27.717 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:27.727 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:27.845 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:27.907 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:47:28.002 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (315 ms) 2024-04-26T04:47:28.002 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-04-26T04:47:28.030 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.044 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:28.170 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:28.232 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:47:28.335 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (334 ms) 2024-04-26T04:47:28.336 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-04-26T04:47:28.365 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.374 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:47:28.380 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.MirroringSamePool (44 ms) 2024-04-26T04:47:28.380 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.MirroringAbort 2024-04-26T04:47:28.407 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.416 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:47:28.422 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.MirroringAbort (43 ms) 2024-04-26T04:47:28.422 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-04-26T04:47:28.450 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.460 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:47:28.466 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (43 ms) 2024-04-26T04:47:28.466 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-04-26T04:47:28.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:28 smithi005 ceph-mon[26371]: pgmap v157: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 496 KiB/s rd, 29 KiB/s wr, 626 op/s 2024-04-26T04:47:28.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:28 smithi005 ceph-mon[36961]: pgmap v157: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 496 KiB/s rd, 29 KiB/s wr, 626 op/s 2024-04-26T04:47:28.493 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.502 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:47:28.508 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (43 ms) 2024-04-26T04:47:28.509 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.MirroringPool 2024-04-26T04:47:28.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:28 smithi111 ceph-mon[30255]: pgmap v157: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 496 KiB/s rd, 29 KiB/s wr, 626 op/s 2024-04-26T04:47:28.538 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.547 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:47:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.MirroringPool (44 ms) 2024-04-26T04:47:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.Group 2024-04-26T04:47:28.580 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.598 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:28.727 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:28.791 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:28.877 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.Group (324 ms) 2024-04-26T04:47:28.878 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.GroupAbort 2024-04-26T04:47:28.904 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:28.921 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:29.057 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:47:29.168 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.GroupAbort (290 ms) 2024-04-26T04:47:29.168 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnaps 2024-04-26T04:47:29.194 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:29.204 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:29.267 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:29.317 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:29.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:29 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3526805879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:29.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:29 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3526805879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:29.485 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:29 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3526805879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:29.597 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:29.705 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnaps (538 ms) 2024-04-26T04:47:29.705 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-04-26T04:47:29.732 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:29.742 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:29.786 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:29.830 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:29.836 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:29.882 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:30.045 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:30.160 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:30.282 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsOtherPool (577 ms) 2024-04-26T04:47:30.282 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-04-26T04:47:30.312 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:30.321 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:30.384 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:30.419 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:30 smithi111 ceph-mon[30255]: pgmap v158: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 21 KiB/s wr, 461 op/s 2024-04-26T04:47:30.438 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:30.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:30 smithi005 ceph-mon[26371]: pgmap v158: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 21 KiB/s wr, 461 op/s 2024-04-26T04:47:30.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:30 smithi005 ceph-mon[36961]: pgmap v158: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 21 KiB/s wr, 461 op/s 2024-04-26T04:47:30.621 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:30.734 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:30.856 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsDataPool (574 ms) 2024-04-26T04:47:30.856 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-04-26T04:47:30.884 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:30.895 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:30.943 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:30.992 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:31.171 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:47:31.178 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:31.285 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:31.391 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (535 ms) 2024-04-26T04:47:31.391 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-04-26T04:47:31.418 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:31.428 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:31.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:31 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/571617881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:31.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:31 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/571617881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:31.487 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:31 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/571617881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:31.537 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 0 2024-04-26T04:47:31.549 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:31.650 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:31.714 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:31.756 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (365 ms) 2024-04-26T04:47:31.756 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-04-26T04:47:31.785 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:31.795 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:31.836 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:31.893 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:32.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:47:32.069 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 0 2024-04-26T04:47:32.081 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:32.160 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:32.214 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (458 ms) 2024-04-26T04:47:32.214 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-04-26T04:47:32.241 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:32.250 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:32.314 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:32.370 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:32 smithi005 ceph-mon[26371]: pgmap v159: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 32 MiB/s wr, 1.94k op/s 2024-04-26T04:47:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:32 smithi005 ceph-mon[26371]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T04:47:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:32 smithi005 ceph-mon[36961]: pgmap v159: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 32 MiB/s wr, 1.94k op/s 2024-04-26T04:47:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:32 smithi005 ceph-mon[36961]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T04:47:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:32 smithi111 ceph-mon[30255]: pgmap v159: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 32 MiB/s wr, 1.94k op/s 2024-04-26T04:47:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:32 smithi111 ceph-mon[30255]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T04:47:32.547 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:47:32.555 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:32.656 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:32.790 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (576 ms) 2024-04-26T04:47:32.790 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-04-26T04:47:32.818 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:32.828 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:32.880 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:32.931 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:33.117 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: after_prepare_snap 2024-04-26T04:47:34.153 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:47:34.172 INFO:tasks.workunit.client.0.smithi005.stdout:write: * 0~1000 2024-04-26T04:47:34.179 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:34.280 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:34.433 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (1643 ms) 2024-04-26T04:47:34.434 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.Snaps 2024-04-26T04:47:34.460 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:34.470 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:34.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:34 smithi005 ceph-mon[26371]: osdmap e115: 8 total, 8 up, 8 in 2024-04-26T04:47:34.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:34 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4124613829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:34.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:34 smithi005 ceph-mon[26371]: pgmap v162: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 33 MiB/s wr, 1.94k op/s 2024-04-26T04:47:34.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:34 smithi005 ceph-mon[36961]: osdmap e115: 8 total, 8 up, 8 in 2024-04-26T04:47:34.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:34 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4124613829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:34.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:34 smithi005 ceph-mon[36961]: pgmap v162: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 33 MiB/s wr, 1.94k op/s 2024-04-26T04:47:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:34 smithi111 ceph-mon[30255]: osdmap e115: 8 total, 8 up, 8 in 2024-04-26T04:47:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:34 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4124613829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:34 smithi111 ceph-mon[30255]: pgmap v162: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 33 MiB/s wr, 1.94k op/s 2024-04-26T04:47:34.526 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:47:35.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:35 smithi005 ceph-mon[26371]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T04:47:35.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:35 smithi005 ceph-mon[36961]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T04:47:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:35 smithi111 ceph-mon[30255]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T04:47:36.162 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:47:36.174 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:47:36.175 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:47:36.175 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:47:36.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:36] "GET /metrics HTTP/1.1" 200 245609 "" "Prometheus/2.43.0" 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[26371]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[26371]: pgmap v165: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4031788596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[36961]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T04:47:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[36961]: pgmap v165: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:36 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4031788596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:36 smithi111 ceph-mon[30255]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T04:47:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:36 smithi111 ceph-mon[30255]: pgmap v165: 137 pgs: 137 active+clean; 145 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:36 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:36 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4031788596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:37.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:37 smithi005 ceph-mon[26371]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T04:47:37.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:37 smithi005 ceph-mon[36961]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T04:47:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:37 smithi111 ceph-mon[30255]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T04:47:38.174 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:47:38.179 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:47:38.179 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:47:38.188 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:47:38.201 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:47:38.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:38 smithi005 ceph-mon[26371]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T04:47:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:38 smithi005 ceph-mon[26371]: pgmap v168: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 44 MiB/s wr, 1.22k op/s 2024-04-26T04:47:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:38 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3732111557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:38 smithi005 ceph-mon[36961]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T04:47:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:38 smithi005 ceph-mon[36961]: pgmap v168: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 44 MiB/s wr, 1.22k op/s 2024-04-26T04:47:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:38 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3732111557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:38 smithi111 ceph-mon[30255]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T04:47:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:38 smithi111 ceph-mon[30255]: pgmap v168: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 44 MiB/s wr, 1.22k op/s 2024-04-26T04:47:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:38 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3732111557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:39.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:39 smithi005 ceph-mon[26371]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T04:47:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:39 smithi005 ceph-mon[36961]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T04:47:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:39 smithi111 ceph-mon[30255]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T04:47:40.177 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:40.199 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:47:40.206 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:40.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:40 smithi005 ceph-mon[26371]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T04:47:40.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:40 smithi005 ceph-mon[26371]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 44 MiB/s wr, 1.22k op/s 2024-04-26T04:47:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:40 smithi005 ceph-mon[36961]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T04:47:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:40 smithi005 ceph-mon[36961]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 44 MiB/s wr, 1.22k op/s 2024-04-26T04:47:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:40 smithi111 ceph-mon[30255]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T04:47:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:40 smithi111 ceph-mon[30255]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 44 MiB/s wr, 1.22k op/s 2024-04-26T04:47:41.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:41 smithi005 ceph-mon[26371]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T04:47:41.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:41 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1112681255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:41 smithi005 ceph-mon[36961]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T04:47:41.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:41 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1112681255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:41.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:41 smithi111 ceph-mon[30255]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T04:47:41.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:41 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1112681255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:42.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:47:42.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:42 smithi005 ceph-mon[26371]: osdmap e123: 8 total, 8 up, 8 in 2024-04-26T04:47:42.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:42 smithi005 ceph-mon[26371]: pgmap v174: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-04-26T04:47:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:42 smithi005 ceph-mon[36961]: osdmap e123: 8 total, 8 up, 8 in 2024-04-26T04:47:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:42 smithi005 ceph-mon[36961]: pgmap v174: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-04-26T04:47:42.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:42 smithi111 ceph-mon[30255]: osdmap e123: 8 total, 8 up, 8 in 2024-04-26T04:47:42.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:42 smithi111 ceph-mon[30255]: pgmap v174: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-04-26T04:47:43.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:43 smithi005 ceph-mon[26371]: osdmap e124: 8 total, 8 up, 8 in 2024-04-26T04:47:43.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:43 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3886553908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:43 smithi005 ceph-mon[36961]: osdmap e124: 8 total, 8 up, 8 in 2024-04-26T04:47:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:43 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3886553908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:43 smithi111 ceph-mon[30255]: osdmap e124: 8 total, 8 up, 8 in 2024-04-26T04:47:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:43 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3886553908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:43.623 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:43.771 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:44.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:44 smithi005 ceph-mon[26371]: osdmap e125: 8 total, 8 up, 8 in 2024-04-26T04:47:44.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:44 smithi005 ceph-mon[26371]: pgmap v177: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-04-26T04:47:44.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:44 smithi005 ceph-mon[36961]: osdmap e125: 8 total, 8 up, 8 in 2024-04-26T04:47:44.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:44 smithi005 ceph-mon[36961]: pgmap v177: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-04-26T04:47:44.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:44 smithi111 ceph-mon[30255]: osdmap e125: 8 total, 8 up, 8 in 2024-04-26T04:47:44.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:44 smithi111 ceph-mon[30255]: pgmap v177: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-04-26T04:47:45.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:45 smithi005 ceph-mon[26371]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T04:47:45.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:45 smithi005 ceph-mon[36961]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T04:47:45.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:45 smithi111 ceph-mon[30255]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T04:47:46.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-mon[26371]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T04:47:46.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2530801383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:46.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-mon[26371]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:47:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-mon[36961]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T04:47:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2530801383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-mon[36961]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:47:46.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:47:46 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:46] "GET /metrics HTTP/1.1" 200 245609 "" "Prometheus/2.43.0" 2024-04-26T04:47:46.507 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.Snaps (12074 ms) 2024-04-26T04:47:46.508 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-04-26T04:47:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:46 smithi111 ceph-mon[30255]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T04:47:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:46 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2530801383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:46 smithi111 ceph-mon[30255]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:47:46.534 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:46.543 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:46.601 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:47:47.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:47 smithi005 ceph-mon[26371]: osdmap e128: 8 total, 8 up, 8 in 2024-04-26T04:47:47.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:47 smithi005 ceph-mon[36961]: osdmap e128: 8 total, 8 up, 8 in 2024-04-26T04:47:47.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:47 smithi111 ceph-mon[30255]: osdmap e128: 8 total, 8 up, 8 in 2024-04-26T04:47:48.223 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:47:48.237 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:47:48.237 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:47:48.237 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:47:48.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:48 smithi005 ceph-mon[26371]: osdmap e129: 8 total, 8 up, 8 in 2024-04-26T04:47:48.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:48 smithi005 ceph-mon[26371]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 440 op/s 2024-04-26T04:47:48.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:48 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1192194685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:48 smithi005 ceph-mon[36961]: osdmap e129: 8 total, 8 up, 8 in 2024-04-26T04:47:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:48 smithi005 ceph-mon[36961]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 440 op/s 2024-04-26T04:47:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:48 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1192194685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:48 smithi111 ceph-mon[30255]: osdmap e129: 8 total, 8 up, 8 in 2024-04-26T04:47:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:48 smithi111 ceph-mon[30255]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 440 op/s 2024-04-26T04:47:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:48 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1192194685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:49.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:49 smithi005 ceph-mon[26371]: osdmap e130: 8 total, 8 up, 8 in 2024-04-26T04:47:49.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:49 smithi005 ceph-mon[36961]: osdmap e130: 8 total, 8 up, 8 in 2024-04-26T04:47:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:49 smithi111 ceph-mon[30255]: osdmap e130: 8 total, 8 up, 8 in 2024-04-26T04:47:50.230 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:47:50.235 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:47:50.236 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:47:50.243 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:47:50.257 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:47:50.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:50 smithi005 ceph-mon[26371]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T04:47:50.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:50 smithi005 ceph-mon[26371]: pgmap v186: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 440 op/s 2024-04-26T04:47:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:50 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3464775886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:50 smithi005 ceph-mon[36961]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T04:47:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:50 smithi005 ceph-mon[36961]: pgmap v186: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 440 op/s 2024-04-26T04:47:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:50 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3464775886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:50.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:50 smithi111 ceph-mon[30255]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T04:47:50.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:50 smithi111 ceph-mon[30255]: pgmap v186: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 440 op/s 2024-04-26T04:47:50.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:50 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3464775886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:51.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:51 smithi005 ceph-mon[26371]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T04:47:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:51 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:51 smithi005 ceph-mon[36961]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T04:47:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:51 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:51 smithi111 ceph-mon[30255]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T04:47:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:51 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:52.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:51 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[32160]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:47:52.127 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:52.150 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:47:52.158 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:52.166 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:47:52.221 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:47:52.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:52 smithi005 ceph-mon[36961]: osdmap e133: 8 total, 8 up, 8 in 2024-04-26T04:47:52.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:52 smithi005 ceph-mon[36961]: pgmap v189: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-04-26T04:47:52.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:52 smithi005 ceph-mon[36961]: osdmap e134: 8 total, 8 up, 8 in 2024-04-26T04:47:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:52 smithi005 ceph-mon[26371]: osdmap e133: 8 total, 8 up, 8 in 2024-04-26T04:47:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:52 smithi005 ceph-mon[26371]: pgmap v189: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-04-26T04:47:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:52 smithi005 ceph-mon[26371]: osdmap e134: 8 total, 8 up, 8 in 2024-04-26T04:47:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:52 smithi111 ceph-mon[30255]: osdmap e133: 8 total, 8 up, 8 in 2024-04-26T04:47:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:52 smithi111 ceph-mon[30255]: pgmap v189: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-04-26T04:47:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:52 smithi111 ceph-mon[30255]: osdmap e134: 8 total, 8 up, 8 in 2024-04-26T04:47:53.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:53 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1797559277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:53.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:53 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1797559277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:53 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1797559277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:54.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:54 smithi005 ceph-mon[26371]: osdmap e135: 8 total, 8 up, 8 in 2024-04-26T04:47:54.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:54 smithi005 ceph-mon[26371]: pgmap v192: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-04-26T04:47:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:54 smithi005 ceph-mon[36961]: osdmap e135: 8 total, 8 up, 8 in 2024-04-26T04:47:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:54 smithi005 ceph-mon[36961]: pgmap v192: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-04-26T04:47:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:54 smithi111 ceph-mon[30255]: osdmap e135: 8 total, 8 up, 8 in 2024-04-26T04:47:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:54 smithi111 ceph-mon[30255]: pgmap v192: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-04-26T04:47:55.475 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:55 smithi111 ceph-mon[30255]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T04:47:55.475 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:55 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:55.475 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:55 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:47:55.475 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:55 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:47:55.475 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:55 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:55.475 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:55 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2806836572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:55.656 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[26371]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2806836572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:55.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[36961]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T04:47:55.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:55.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:47:55.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:47:55.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:55.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:55 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2806836572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:55.812 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:47:56.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:47:56] "GET /metrics HTTP/1.1" 200 245719 "" "Prometheus/2.43.0" 2024-04-26T04:47:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[36961]: Upgrade: Updating mgr.x 2024-04-26T04:47:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[36961]: Deploying daemon mgr.x on smithi111 2024-04-26T04:47:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[36961]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T04:47:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[36961]: pgmap v195: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:47:56.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[26371]: Upgrade: Updating mgr.x 2024-04-26T04:47:56.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[26371]: Deploying daemon mgr.x on smithi111 2024-04-26T04:47:56.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[26371]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T04:47:56.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:56 smithi005 ceph-mon[26371]: pgmap v195: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:47:56.502 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:56 smithi111 ceph-mon[30255]: Upgrade: Updating mgr.x 2024-04-26T04:47:56.502 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:56 smithi111 ceph-mon[30255]: Deploying daemon mgr.x on smithi111 2024-04-26T04:47:56.502 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:56 smithi111 ceph-mon[30255]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T04:47:56.502 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:56 smithi111 ceph-mon[30255]: pgmap v195: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:47:56.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:56 smithi111 systemd[1]: Stopping Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:47:57.076 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:56 smithi111 podman[63524]: 2024-04-26 04:47:56.770695162 +0000 UTC m=+0.127976792 container died 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T04:47:57.418 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:57 smithi111 ceph-mon[30255]: osdmap e138: 8 total, 8 up, 8 in 2024-04-26T04:47:57.419 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 podman[63524]: 2024-04-26 04:47:57.199808189 +0000 UTC m=+0.557089813 container cleanup 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T04:47:57.419 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 bash[63524]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:47:57.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:57 smithi005 ceph-mon[26371]: osdmap e138: 8 total, 8 up, 8 in 2024-04-26T04:47:57.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:57 smithi005 ceph-mon[36961]: osdmap e138: 8 total, 8 up, 8 in 2024-04-26T04:47:57.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 podman[63538]: 2024-04-26 04:47:57.418319761 +0000 UTC m=+0.641143159 container remove 587d3c982d8c6c6c38e46b75aeebc0df3802251271d5fe9a86d12625dca7a1f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T04:47:57.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:47:57.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:47:57.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 systemd[1]: Stopped Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:47:57.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.447s CPU time. 2024-04-26T04:47:58.169 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:57 smithi111 systemd[1]: Starting Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:47:58.169 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 podman[63668]: 2024-04-26 04:47:57.953138662 +0000 UTC m=+0.018113959 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:58.169 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 podman[63668]: 2024-04-26 04:47:58.061024565 +0000 UTC m=+0.125999857 container create 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T04:47:58.453 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-mon[30255]: osdmap e139: 8 total, 8 up, 8 in 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1162943351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-mon[30255]: pgmap v198: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 497 op/s 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 podman[63668]: 2024-04-26 04:47:58.288289554 +0000 UTC m=+0.353264853 container init 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 podman[63668]: 2024-04-26 04:47:58.291321164 +0000 UTC m=+0.356296457 container start 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-mgr[63687]: -- 172.21.15.111:0/3237963070 <== mon.1 v2:172.21.15.111:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b11704c5a0 con 0x55b117020c00 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 bash[63668]: 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 2024-04-26T04:47:58.454 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 systemd[1]: Started Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:47:58.621 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsOtherPool (12114 ms) 2024-04-26T04:47:58.622 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-04-26T04:47:58.656 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:47:58.667 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:47:58.725 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:47:58.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:58 smithi005 ceph-mon[36961]: osdmap e139: 8 total, 8 up, 8 in 2024-04-26T04:47:58.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:58 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1162943351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:58.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:58 smithi005 ceph-mon[36961]: pgmap v198: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 497 op/s 2024-04-26T04:47:58.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:58 smithi005 ceph-mon[26371]: osdmap e139: 8 total, 8 up, 8 in 2024-04-26T04:47:58.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:58 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1162943351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:58.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:58 smithi005 ceph-mon[26371]: pgmap v198: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 497 op/s 2024-04-26T04:47:58.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:47:58.452+0000 7fdbf65ad200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:47:58.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:47:58.656+0000 7fdbf65ad200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:47:59.070 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:47:58.790+0000 7fdbf65ad200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[26371]: osdmap e140: 8 total, 8 up, 8 in 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[36961]: osdmap e140: 8 total, 8 up, 8 in 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:47:59 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:59 smithi111 ceph-mon[30255]: osdmap e140: 8 total, 8 up, 8 in 2024-04-26T04:47:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:59 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:59 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:47:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:47:59 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:59.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:47:59.113+0000 7fdbf65ad200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:47:59.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:47:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:47:59.203+0000 7fdbf65ad200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:48:00.462 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:48:00.482 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:48:00.482 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:48:00.482 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:48:00.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:00 smithi111 ceph-mon[30255]: osdmap e141: 8 total, 8 up, 8 in 2024-04-26T04:48:00.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:00 smithi111 ceph-mon[30255]: pgmap v201: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 497 op/s 2024-04-26T04:48:00.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:00 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/795654870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:00 smithi005 ceph-mon[26371]: osdmap e141: 8 total, 8 up, 8 in 2024-04-26T04:48:00.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:00 smithi005 ceph-mon[26371]: pgmap v201: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 497 op/s 2024-04-26T04:48:00.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:00 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/795654870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:00 smithi005 ceph-mon[36961]: osdmap e141: 8 total, 8 up, 8 in 2024-04-26T04:48:00.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:00 smithi005 ceph-mon[36961]: pgmap v201: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 497 op/s 2024-04-26T04:48:00.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:00 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/795654870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.783 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:00.517+0000 7fdbf65ad200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:48:00.783 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:00.696+0000 7fdbf65ad200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:01.189 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:00.875+0000 7fdbf65ad200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:48:01.450 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:01.187+0000 7fdbf65ad200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:48:01.450 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:01.282+0000 7fdbf65ad200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:48:01.703 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-mon[30255]: osdmap e142: 8 total, 8 up, 8 in 2024-04-26T04:48:01.703 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:01.703 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:01.703 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:01.501+0000 7fdbf65ad200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:48:01.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:01 smithi005 ceph-mon[36961]: osdmap e142: 8 total, 8 up, 8 in 2024-04-26T04:48:01.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:01 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:01.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:01 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:01.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:01 smithi005 ceph-mon[26371]: osdmap e142: 8 total, 8 up, 8 in 2024-04-26T04:48:01.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:01 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:01.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:01 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:02.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:01.701+0000 7fdbf65ad200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:02.127 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:48:02.132 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:48:02.132 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:48:02.141 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:48:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:48:02.458 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:02.335+0000 7fdbf65ad200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:48:02.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[26371]: osdmap e143: 8 total, 8 up, 8 in 2024-04-26T04:48:02.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[26371]: pgmap v204: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-04-26T04:48:02.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[26371]: osdmap e144: 8 total, 8 up, 8 in 2024-04-26T04:48:02.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/857338540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[36961]: osdmap e143: 8 total, 8 up, 8 in 2024-04-26T04:48:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[36961]: pgmap v204: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-04-26T04:48:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[36961]: osdmap e144: 8 total, 8 up, 8 in 2024-04-26T04:48:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:02 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/857338540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:02.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-mon[30255]: osdmap e143: 8 total, 8 up, 8 in 2024-04-26T04:48:02.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-mon[30255]: pgmap v204: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-04-26T04:48:02.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-mon[30255]: osdmap e144: 8 total, 8 up, 8 in 2024-04-26T04:48:02.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/857338540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:03.122 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:02.797+0000 7fdbf65ad200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:48:03.122 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:02.901+0000 7fdbf65ad200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:48:03.122 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:03.121+0000 7fdbf65ad200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:48:03.929 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.929 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.929 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-mon[30255]: osdmap e145: 8 total, 8 up, 8 in 2024-04-26T04:48:03.929 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.929 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.929 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-mon[30255]: pgmap v207: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-04-26T04:48:03.930 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:03.677+0000 7fdbf65ad200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:48:03.930 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:03.783+0000 7fdbf65ad200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[36961]: osdmap e145: 8 total, 8 up, 8 in 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[36961]: pgmap v207: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[26371]: osdmap e145: 8 total, 8 up, 8 in 2024-04-26T04:48:03.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:03.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:03 smithi005 ceph-mon[26371]: pgmap v207: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-04-26T04:48:04.132 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:04.154 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:48:04.161 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:48:04.183 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:04.035+0000 7fdbf65ad200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:48:04.496 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:04.181+0000 7fdbf65ad200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:48:04.497 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:04.282+0000 7fdbf65ad200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:48:04.497 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:04.401+0000 7fdbf65ad200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:04.497 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:04.495+0000 7fdbf65ad200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:48:05.427 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-mon[30255]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T04:48:05.427 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2757187889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:05.428 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:05.335+0000 7fdbf65ad200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:48:05.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:05 smithi005 ceph-mon[36961]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T04:48:05.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:05 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2757187889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:05 smithi005 ceph-mon[26371]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T04:48:05.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:05 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2757187889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:05.678 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:05.568+0000 7fdbf65ad200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:48:05.678 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:05] ENGINE Bus STARTING 2024-04-26T04:48:06.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: CherryPy Checker: 2024-04-26T04:48:06.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: The Application mounted at '' has an empty config. 2024-04-26T04:48:06.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:06.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:05] ENGINE Serving on http://:::9283 2024-04-26T04:48:06.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:05] ENGINE Bus STARTED 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:06] "GET /metrics HTTP/1.1" 200 245810 "" "Prometheus/2.43.0" 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: pgmap v210: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: Standby manager daemon x restarted 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: Standby manager daemon x started 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:48:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: pgmap v210: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: Standby manager daemon x restarted 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: Standby manager daemon x started 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:48:06.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:06 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: pgmap v210: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: Standby manager daemon x restarted 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: Standby manager daemon x started 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:48:06.489 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:06 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:48:07.263 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: mgrmap e22: y(active, since 4m), standbys: x 2024-04-26T04:48:07.263 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T04:48:07.263 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/428784105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:07.263 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-mon[30255]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:07] ENGINE Bus STOPPING 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:07] ENGINE Bus STOPPED 2024-04-26T04:48:07.264 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:07] ENGINE Bus STARTING 2024-04-26T04:48:07.381 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:07.173+0000 7ff7fc304700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:48:07.381 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setuser ceph since I am not root 2024-04-26T04:48:07.381 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ignoring --setgroup ceph since I am not root 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mgr[26603]: -- 172.21.15.5:0/320471147 <== mon.0 v2:172.21.15.5:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563e482009c0 con 0x563e481d6800 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:07.376+0000 7f02f09be1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: mgrmap e22: y(active, since 4m), standbys: x 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/428784105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:07.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[26371]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: mgrmap e22: y(active, since 4m), standbys: x 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/428784105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' 2024-04-26T04:48:07.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-mon[36961]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T04:48:07.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.321+0000 7fdb5719e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.321+0000 7fdb5719e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.321+0000 7fdb5719e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.321+0000 7fdb5719e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.321+0000 7fdb5719e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.322+0000 7fdb53997700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.322+0000 7fdb53997700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.322+0000 7fdb53997700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.322+0000 7fdb53997700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:48:07.322+0000 7fdb53997700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:07] ENGINE Serving on http://:::9283 2024-04-26T04:48:07.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:07] ENGINE Bus STARTED 2024-04-26T04:48:07.582 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:48:07.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:07.593+0000 7f02f09be1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:07.753 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:48:08.066 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:07.738+0000 7f02f09be1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:48:08.066 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:08.065+0000 7f02f09be1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:48:08.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:08.151+0000 7f02f09be1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: Activating manager daemon x 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: mgrmap e23: x(active, starting, since 0.00986064s) 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:48:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: Manager daemon x is now available 2024-04-26T04:48:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:48:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: Activating manager daemon x 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: mgrmap e23: x(active, starting, since 0.00986064s) 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:48:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: Manager daemon x is now available 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:08.484 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:08 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:48:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: Activating manager daemon x 2024-04-26T04:48:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24335 172.21.15.5:0/4143084226' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: mgrmap e23: x(active, starting, since 0.00986064s) 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:48:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: Manager daemon x is now available 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:08.520 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:08 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:09.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:09.379+0000 7f02f09be1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[36961]: Deploying cephadm binary to smithi111 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[36961]: Deploying cephadm binary to smithi005 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[36961]: mgrmap e24: x(active, since 1.00797s) 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[36961]: pgmap v3: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[36961]: [26/Apr/2024:04:48:08] ENGINE Bus STARTING 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[36961]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[26371]: Deploying cephadm binary to smithi111 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[26371]: Deploying cephadm binary to smithi005 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[26371]: mgrmap e24: x(active, since 1.00797s) 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[26371]: pgmap v3: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[26371]: [26/Apr/2024:04:48:08] ENGINE Bus STARTING 2024-04-26T04:48:09.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-mon[26371]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T04:48:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:09 smithi111 ceph-mon[30255]: Deploying cephadm binary to smithi111 2024-04-26T04:48:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:09 smithi111 ceph-mon[30255]: Deploying cephadm binary to smithi005 2024-04-26T04:48:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:09 smithi111 ceph-mon[30255]: mgrmap e24: x(active, since 1.00797s) 2024-04-26T04:48:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:09 smithi111 ceph-mon[30255]: pgmap v3: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:09 smithi111 ceph-mon[30255]: [26/Apr/2024:04:48:08] ENGINE Bus STARTING 2024-04-26T04:48:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:09 smithi111 ceph-mon[30255]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T04:48:09.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:09.551+0000 7f02f09be1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:09.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:09 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:09.726+0000 7f02f09be1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:48:10.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:10.007+0000 7f02f09be1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:48:10.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:10.100+0000 7f02f09be1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:48:10.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:10.201+0000 7f02f09be1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:48:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[26371]: [26/Apr/2024:04:48:08] ENGINE Serving on https://172.21.15.111:7150 2024-04-26T04:48:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[26371]: [26/Apr/2024:04:48:08] ENGINE Serving on http://172.21.15.111:8765 2024-04-26T04:48:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[26371]: [26/Apr/2024:04:48:08] ENGINE Bus STARTED 2024-04-26T04:48:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[26371]: pgmap v5: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1089965230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[26371]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T04:48:10.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[36961]: [26/Apr/2024:04:48:08] ENGINE Serving on https://172.21.15.111:7150 2024-04-26T04:48:10.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[36961]: [26/Apr/2024:04:48:08] ENGINE Serving on http://172.21.15.111:8765 2024-04-26T04:48:10.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[36961]: [26/Apr/2024:04:48:08] ENGINE Bus STARTED 2024-04-26T04:48:10.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[36961]: pgmap v5: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:10.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1089965230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:10.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-mon[36961]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T04:48:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:10 smithi111 ceph-mon[30255]: [26/Apr/2024:04:48:08] ENGINE Serving on https://172.21.15.111:7150 2024-04-26T04:48:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:10 smithi111 ceph-mon[30255]: [26/Apr/2024:04:48:08] ENGINE Serving on http://172.21.15.111:8765 2024-04-26T04:48:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:10 smithi111 ceph-mon[30255]: [26/Apr/2024:04:48:08] ENGINE Bus STARTED 2024-04-26T04:48:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:10 smithi111 ceph-mon[30255]: pgmap v5: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:10 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1089965230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:10.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:10 smithi111 ceph-mon[30255]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T04:48:10.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:10.488+0000 7f02f09be1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:11.034 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsDataPool (12413 ms) 2024-04-26T04:48:11.034 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-04-26T04:48:11.062 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:11.073 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:48:11.129 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:48:11.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-mon[36961]: mgrmap e25: x(active, since 3s) 2024-04-26T04:48:11.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-mon[36961]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T04:48:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-mon[26371]: mgrmap e25: x(active, since 3s) 2024-04-26T04:48:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-mon[26371]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T04:48:11.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:11.013+0000 7f02f09be1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:48:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:11 smithi111 ceph-mon[30255]: mgrmap e25: x(active, since 3s) 2024-04-26T04:48:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:11 smithi111 ceph-mon[30255]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T04:48:11.877 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:11.585+0000 7f02f09be1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:48:11.877 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:11.684+0000 7f02f09be1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:48:11.877 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:11.876+0000 7f02f09be1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:48:12.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:11] "GET /metrics HTTP/1.1" 200 35747 "" "Prometheus/2.43.0" 2024-04-26T04:48:12.453 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-mon[36961]: pgmap v8: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:12.454 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-mon[36961]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T04:48:12.454 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4243162285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:12.454 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-mon[26371]: pgmap v8: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:12.454 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-mon[26371]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T04:48:12.454 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4243162285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:12.454 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:12.185+0000 7f02f09be1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:48:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:12 smithi111 ceph-mon[30255]: pgmap v8: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:12 smithi111 ceph-mon[30255]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T04:48:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:12 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4243162285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:12.452+0000 7f02f09be1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:48:12.769 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:48:12.787 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:48:12.787 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:48:12.787 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:48:12.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:12.713+0000 7f02f09be1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:48:12.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:12.893+0000 7f02f09be1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:48:13.434 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:13.002+0000 7f02f09be1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:48:13.434 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:13.140+0000 7f02f09be1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:13.434 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:13.248+0000 7f02f09be1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:48:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:13 smithi111 ceph-mon[30255]: osdmap e155: 8 total, 8 up, 8 in 2024-04-26T04:48:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:13 smithi111 ceph-mon[30255]: pgmap v11: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 1 op/s 2024-04-26T04:48:14.156 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-mon[26371]: osdmap e155: 8 total, 8 up, 8 in 2024-04-26T04:48:14.156 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-mon[26371]: pgmap v11: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 1 op/s 2024-04-26T04:48:14.157 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-mon[36961]: osdmap e155: 8 total, 8 up, 8 in 2024-04-26T04:48:14.157 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:13 smithi005 ceph-mon[36961]: pgmap v11: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 1 op/s 2024-04-26T04:48:14.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:14.154+0000 7f02f09be1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:48:14.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:14.386+0000 7f02f09be1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:48:14.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:48:14] ENGINE Bus STARTING 2024-04-26T04:48:14.759 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: CherryPy Checker: 2024-04-26T04:48:14.759 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: The Application mounted at '' has an empty config. 2024-04-26T04:48:14.759 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: 2024-04-26T04:48:14.760 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:48:14] ENGINE Serving on http://:::9283 2024-04-26T04:48:14.760 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: [26/Apr/2024:04:48:14] ENGINE Bus STARTED 2024-04-26T04:48:14.778 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:48:14.789 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:48:14.789 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:48:14.797 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:48:14.815 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: osdmap e156: 8 total, 8 up, 8 in 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2030552004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: Standby manager daemon y started 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:48:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:14 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:48:15.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: osdmap e156: 8 total, 8 up, 8 in 2024-04-26T04:48:15.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2030552004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: Standby manager daemon y started 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: osdmap e156: 8 total, 8 up, 8 in 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2030552004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: Standby manager daemon y started 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:48:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:48:15.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:48:15.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:14 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/1561421699' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:48:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:15 smithi111 ceph-mon[30255]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:48:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:15 smithi111 ceph-mon[30255]: osdmap e157: 8 total, 8 up, 8 in 2024-04-26T04:48:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:15 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:48:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:15 smithi111 ceph-mon[30255]: pgmap v14: 137 pgs: 137 active+clean; 425 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 173 op/s 2024-04-26T04:48:16.033 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[36961]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:48:16.033 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[36961]: osdmap e157: 8 total, 8 up, 8 in 2024-04-26T04:48:16.033 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:48:16.034 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[36961]: pgmap v14: 137 pgs: 137 active+clean; 425 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 173 op/s 2024-04-26T04:48:16.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[26371]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:48:16.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[26371]: osdmap e157: 8 total, 8 up, 8 in 2024-04-26T04:48:16.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:48:16.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:15 smithi005 ceph-mon[26371]: pgmap v14: 137 pgs: 137 active+clean; 425 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 173 op/s 2024-04-26T04:48:16.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:48:16.793 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:16.814 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:48:16.821 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:48:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:16 smithi111 ceph-mon[30255]: osdmap e158: 8 total, 8 up, 8 in 2024-04-26T04:48:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:16 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:16 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:16 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/661103484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[26371]: osdmap e158: 8 total, 8 up, 8 in 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/661103484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[36961]: osdmap e158: 8 total, 8 up, 8 in 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:16 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/661103484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:18.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: osdmap e159: 8 total, 8 up, 8 in 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi005 to 2481M 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: osdmap e160: 8 total, 8 up, 8 in 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[26371]: pgmap v18: 137 pgs: 137 active+clean; 425 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: osdmap e159: 8 total, 8 up, 8 in 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi005 to 2481M 2024-04-26T04:48:18.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: osdmap e160: 8 total, 8 up, 8 in 2024-04-26T04:48:18.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:17 smithi005 ceph-mon[36961]: pgmap v18: 137 pgs: 137 active+clean; 425 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: osdmap e159: 8 total, 8 up, 8 in 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:18.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi005 to 2481M 2024-04-26T04:48:18.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:18.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: osdmap e160: 8 total, 8 up, 8 in 2024-04-26T04:48:18.271 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:17 smithi111 ceph-mon[30255]: pgmap v18: 137 pgs: 137 active+clean; 425 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:48:19.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:19 smithi005 ceph-mon[26371]: osdmap e161: 8 total, 8 up, 8 in 2024-04-26T04:48:19.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:19 smithi005 ceph-mon[36961]: osdmap e161: 8 total, 8 up, 8 in 2024-04-26T04:48:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:19 smithi111 ceph-mon[30255]: osdmap e161: 8 total, 8 up, 8 in 2024-04-26T04:48:19.553 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:48:19.617 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:48:19.752 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:48:20.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:20 smithi005 ceph-mon[26371]: osdmap e162: 8 total, 8 up, 8 in 2024-04-26T04:48:20.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:20 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1163034747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:20.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:20 smithi005 ceph-mon[26371]: pgmap v21: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 142 KiB/s rd, 4.7 MiB/s wr, 203 op/s 2024-04-26T04:48:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:20 smithi005 ceph-mon[36961]: osdmap e162: 8 total, 8 up, 8 in 2024-04-26T04:48:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:20 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1163034747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:20 smithi005 ceph-mon[36961]: pgmap v21: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 142 KiB/s rd, 4.7 MiB/s wr, 203 op/s 2024-04-26T04:48:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:20 smithi111 ceph-mon[30255]: osdmap e162: 8 total, 8 up, 8 in 2024-04-26T04:48:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:20 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1163034747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:20 smithi111 ceph-mon[30255]: pgmap v21: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 142 KiB/s rd, 4.7 MiB/s wr, 203 op/s 2024-04-26T04:48:21.251 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:21 smithi111 ceph-mon[30255]: osdmap e163: 8 total, 8 up, 8 in 2024-04-26T04:48:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:21 smithi005 ceph-mon[26371]: osdmap e163: 8 total, 8 up, 8 in 2024-04-26T04:48:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:21 smithi005 ceph-mon[36961]: osdmap e163: 8 total, 8 up, 8 in 2024-04-26T04:48:22.002 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:21] "GET /metrics HTTP/1.1" 200 35747 "" "Prometheus/2.43.0" 2024-04-26T04:48:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:22 smithi111 ceph-mon[30255]: osdmap e164: 8 total, 8 up, 8 in 2024-04-26T04:48:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:22 smithi111 ceph-mon[30255]: pgmap v24: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 4.0 MiB/s wr, 173 op/s 2024-04-26T04:48:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:22 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3823663737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:22.419 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (11385 ms) 2024-04-26T04:48:22.420 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-04-26T04:48:22.450 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:22.462 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:48:22.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:22 smithi005 ceph-mon[26371]: osdmap e164: 8 total, 8 up, 8 in 2024-04-26T04:48:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:22 smithi005 ceph-mon[26371]: pgmap v24: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 4.0 MiB/s wr, 173 op/s 2024-04-26T04:48:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:22 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3823663737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:22 smithi005 ceph-mon[36961]: osdmap e164: 8 total, 8 up, 8 in 2024-04-26T04:48:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:22 smithi005 ceph-mon[36961]: pgmap v24: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 4.0 MiB/s wr, 173 op/s 2024-04-26T04:48:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:22 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3823663737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:22.523 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:48:23.470 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:23 smithi111 ceph-mon[30255]: osdmap e165: 8 total, 8 up, 8 in 2024-04-26T04:48:23.470 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:23 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:23.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:23 smithi005 ceph-mon[26371]: osdmap e165: 8 total, 8 up, 8 in 2024-04-26T04:48:23.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:23 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:23 smithi005 ceph-mon[36961]: osdmap e165: 8 total, 8 up, 8 in 2024-04-26T04:48:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:23 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:24.172 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:48:24.193 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:48:24.193 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:48:24.193 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:48:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[36961]: osdmap e166: 8 total, 8 up, 8 in 2024-04-26T04:48:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[36961]: pgmap v27: 137 pgs: 137 active+clean; 445 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 5.7 KiB/s wr, 40 op/s 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/27032490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[26371]: osdmap e166: 8 total, 8 up, 8 in 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[26371]: pgmap v27: 137 pgs: 137 active+clean; 445 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 5.7 KiB/s wr, 40 op/s 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:24 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/27032490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:24 smithi111 ceph-mon[30255]: osdmap e166: 8 total, 8 up, 8 in 2024-04-26T04:48:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:24 smithi111 ceph-mon[30255]: pgmap v27: 137 pgs: 137 active+clean; 445 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 5.7 KiB/s wr, 40 op/s 2024-04-26T04:48:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:24 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:24 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:24 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/27032490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:25.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:25 smithi005 ceph-mon[26371]: osdmap e167: 8 total, 8 up, 8 in 2024-04-26T04:48:25.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:25 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:25.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:25 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:25 smithi005 ceph-mon[36961]: osdmap e167: 8 total, 8 up, 8 in 2024-04-26T04:48:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:25 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:25 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:25 smithi111 ceph-mon[30255]: osdmap e167: 8 total, 8 up, 8 in 2024-04-26T04:48:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:25 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:25 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:26.185 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:48:26.192 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:48:26.192 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:48:26.200 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:48:26.213 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:48:26.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:26 smithi005 ceph-mon[26371]: osdmap e168: 8 total, 8 up, 8 in 2024-04-26T04:48:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:26 smithi005 ceph-mon[26371]: pgmap v30: 137 pgs: 137 active+clean; 465 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 521 op/s 2024-04-26T04:48:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:26 smithi005 ceph-mon[36961]: osdmap e168: 8 total, 8 up, 8 in 2024-04-26T04:48:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:26 smithi005 ceph-mon[36961]: pgmap v30: 137 pgs: 137 active+clean; 465 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 521 op/s 2024-04-26T04:48:26.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:26 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:48:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:26 smithi111 ceph-mon[30255]: osdmap e168: 8 total, 8 up, 8 in 2024-04-26T04:48:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:26 smithi111 ceph-mon[30255]: pgmap v30: 137 pgs: 137 active+clean; 465 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 521 op/s 2024-04-26T04:48:27.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:27 smithi005 ceph-mon[26371]: osdmap e169: 8 total, 8 up, 8 in 2024-04-26T04:48:27.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:27 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2061701011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:27.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:27 smithi005 ceph-mon[26371]: osdmap e170: 8 total, 8 up, 8 in 2024-04-26T04:48:27.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:27 smithi005 ceph-mon[36961]: osdmap e169: 8 total, 8 up, 8 in 2024-04-26T04:48:27.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:27 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2061701011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:27.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:27 smithi005 ceph-mon[36961]: osdmap e170: 8 total, 8 up, 8 in 2024-04-26T04:48:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:27 smithi111 ceph-mon[30255]: osdmap e169: 8 total, 8 up, 8 in 2024-04-26T04:48:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:27 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2061701011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:27 smithi111 ceph-mon[30255]: osdmap e170: 8 total, 8 up, 8 in 2024-04-26T04:48:28.136 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:28.157 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:48:28.165 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 0 2024-04-26T04:48:28.183 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:48:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:28 smithi111 ceph-mon[30255]: pgmap v33: 137 pgs: 137 active+clean; 465 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 481 op/s 2024-04-26T04:48:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:28 smithi111 ceph-mon[30255]: osdmap e171: 8 total, 8 up, 8 in 2024-04-26T04:48:28.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:28 smithi005 ceph-mon[26371]: pgmap v33: 137 pgs: 137 active+clean; 465 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 481 op/s 2024-04-26T04:48:28.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:28 smithi005 ceph-mon[26371]: osdmap e171: 8 total, 8 up, 8 in 2024-04-26T04:48:28.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:28 smithi005 ceph-mon[36961]: pgmap v33: 137 pgs: 137 active+clean; 465 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 481 op/s 2024-04-26T04:48:28.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:28 smithi005 ceph-mon[36961]: osdmap e171: 8 total, 8 up, 8 in 2024-04-26T04:48:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Adjusting osd_memory_target on smithi111 to 2480M 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:48:29.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/689470905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: osdmap e172: 8 total, 8 up, 8 in 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: pgmap v36: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 8.0 MiB/s wr, 174 op/s 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Adjusting osd_memory_target on smithi111 to 2480M 2024-04-26T04:48:29.733 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/689470905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: osdmap e172: 8 total, 8 up, 8 in 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: pgmap v36: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 8.0 MiB/s wr, 174 op/s 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:29 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Adjusting osd_memory_target on smithi111 to 2480M 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/689470905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:29.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: osdmap e172: 8 total, 8 up, 8 in 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: pgmap v36: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 8.0 MiB/s wr, 174 op/s 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:29.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:29 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: Reconfiguring daemon alertmanager.a on smithi005 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[26371]: osdmap e173: 8 total, 8 up, 8 in 2024-04-26T04:48:30.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:48:30.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: Reconfiguring daemon alertmanager.a on smithi005 2024-04-26T04:48:30.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:30 smithi005 ceph-mon[36961]: osdmap e173: 8 total, 8 up, 8 in 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: Reconfiguring daemon alertmanager.a on smithi005 2024-04-26T04:48:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:30 smithi111 ceph-mon[30255]: osdmap e173: 8 total, 8 up, 8 in 2024-04-26T04:48:31.558 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:48:31.694 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:48:31.939 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:31] "GET /metrics HTTP/1.1" 200 38777 "" "Prometheus/2.43.0" 2024-04-26T04:48:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:32 smithi111 ceph-mon[30255]: osdmap e174: 8 total, 8 up, 8 in 2024-04-26T04:48:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:32 smithi111 ceph-mon[30255]: pgmap v39: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 8.0 MiB/s wr, 174 op/s 2024-04-26T04:48:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:32 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/741688473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:32 smithi005 ceph-mon[26371]: osdmap e174: 8 total, 8 up, 8 in 2024-04-26T04:48:32.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:32 smithi005 ceph-mon[26371]: pgmap v39: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 8.0 MiB/s wr, 174 op/s 2024-04-26T04:48:32.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:32 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/741688473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:32 smithi005 ceph-mon[36961]: osdmap e174: 8 total, 8 up, 8 in 2024-04-26T04:48:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:32 smithi005 ceph-mon[36961]: pgmap v39: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 8.0 MiB/s wr, 174 op/s 2024-04-26T04:48:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:32 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/741688473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:33.187 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:32 smithi005 systemd[1]: Stopping Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:33.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:33 smithi005 ceph-mon[26371]: osdmap e175: 8 total, 8 up, 8 in 2024-04-26T04:48:33.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:33 smithi005 ceph-mon[36961]: osdmap e175: 8 total, 8 up, 8 in 2024-04-26T04:48:33.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[70833]: ts=2024-04-26T04:48:33.208Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:48:33.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:33 smithi005 podman[81971]: 2024-04-26 04:48:33.248715827 +0000 UTC m=+0.170599865 container died 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:33.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:33 smithi111 ceph-mon[30255]: osdmap e175: 8 total, 8 up, 8 in 2024-04-26T04:48:33.793 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:33 smithi005 podman[81971]: 2024-04-26 04:48:33.599954716 +0000 UTC m=+0.521838739 container cleanup 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:33.793 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:33 smithi005 bash[81971]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:48:34.159 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:33 smithi005 podman[81985]: 2024-04-26 04:48:33.793169794 +0000 UTC m=+0.535839312 container remove 8b891aba4f92ebcd0753c82125f420efa281a93fbe7510f9a24a8054cd3cbd7b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:34.159 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:33 smithi005 podman[81985]: 2024-04-26 04:48:33.947097369 +0000 UTC m=+0.689766899 volume remove bda220482b45896b851925ff39aa4aca40558f4786c8a3226c64c4eba7097cd4 2024-04-26T04:48:34.462 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-mon[26371]: pgmap v41: 137 pgs: 137 active+clean; 509 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 198 op/s 2024-04-26T04:48:34.462 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-mon[26371]: osdmap e176: 8 total, 8 up, 8 in 2024-04-26T04:48:34.462 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2742626089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:34.462 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-mon[36961]: pgmap v41: 137 pgs: 137 active+clean; 509 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 198 op/s 2024-04-26T04:48:34.463 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-mon[36961]: osdmap e176: 8 total, 8 up, 8 in 2024-04-26T04:48:34.463 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2742626089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:34.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:48:34.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 systemd[1]: Stopped Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:34.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.072s CPU time. 2024-04-26T04:48:34.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 systemd[1]: Starting Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:34.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 podman[82100]: 2024-04-26 04:48:34.331790535 +0000 UTC m=+0.022479541 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:48:34.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 podman[82100]: 2024-04-26 04:48:34.461744052 +0000 UTC m=+0.152433048 volume create afa5c254e90b90d09fd308b988832712c8e35e51b48a9e93d3f9c0905bda2792 2024-04-26T04:48:34.466 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (12047 ms) 2024-04-26T04:48:34.466 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-04-26T04:48:34.497 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:34.510 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:48:34.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:34 smithi111 ceph-mon[30255]: pgmap v41: 137 pgs: 137 active+clean; 509 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 4.0 MiB/s wr, 198 op/s 2024-04-26T04:48:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:34 smithi111 ceph-mon[30255]: osdmap e176: 8 total, 8 up, 8 in 2024-04-26T04:48:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:34 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2742626089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:34.573 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:48:34.729 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 podman[82100]: 2024-04-26 04:48:34.562489606 +0000 UTC m=+0.253178602 container create 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:35.205 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 podman[82100]: 2024-04-26 04:48:34.890507607 +0000 UTC m=+0.581196604 container init 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:35.205 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 podman[82100]: 2024-04-26 04:48:34.893140122 +0000 UTC m=+0.583829122 container start 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:35.205 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.905Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.905Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.906Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.5 port=9094 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.907Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.930Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.931Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.933Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:34.934Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 bash[82100]: 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f 2024-04-26T04:48:35.206 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:34 smithi005 systemd[1]: Started Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:35.476 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:35 smithi005 ceph-mon[36961]: osdmap e177: 8 total, 8 up, 8 in 2024-04-26T04:48:35.476 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:35 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:35.476 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:35 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:35.476 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:35 smithi005 ceph-mon[26371]: osdmap e177: 8 total, 8 up, 8 in 2024-04-26T04:48:35.476 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:35 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:35.476 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:35 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:35 smithi111 ceph-mon[30255]: osdmap e177: 8 total, 8 up, 8 in 2024-04-26T04:48:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:35 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:35 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:35.729 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:35 smithi005 systemd[1]: Stopping Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:36.154 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:35 smithi005 podman[82260]: 2024-04-26 04:48:35.881118416 +0000 UTC m=+0.128149731 container died 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:36.154 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[26599]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:48:36.227 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:48:36.240 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:48:36.240 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:48:36.241 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:48:36.479 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 podman[82260]: 2024-04-26 04:48:36.336061356 +0000 UTC m=+0.583092679 container cleanup 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:36.479 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 bash[82260]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[26371]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[26371]: Reconfiguring daemon node-exporter.a on smithi005 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[26371]: pgmap v44: 137 pgs: 137 active+clean; 521 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 434 op/s 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[26371]: osdmap e178: 8 total, 8 up, 8 in 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/479504753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[36961]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:48:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[36961]: Reconfiguring daemon node-exporter.a on smithi005 2024-04-26T04:48:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[36961]: pgmap v44: 137 pgs: 137 active+clean; 521 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 434 op/s 2024-04-26T04:48:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[36961]: osdmap e178: 8 total, 8 up, 8 in 2024-04-26T04:48:36.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/479504753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:36 smithi111 ceph-mon[30255]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:48:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:36 smithi111 ceph-mon[30255]: Reconfiguring daemon node-exporter.a on smithi005 2024-04-26T04:48:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:36 smithi111 ceph-mon[30255]: pgmap v44: 137 pgs: 137 active+clean; 521 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 434 op/s 2024-04-26T04:48:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:36 smithi111 ceph-mon[30255]: osdmap e178: 8 total, 8 up, 8 in 2024-04-26T04:48:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:36 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/479504753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:36.744 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 podman[82293]: 2024-04-26 04:48:36.621106025 +0000 UTC m=+0.744956158 container remove 12c05c1a199c91e0cb70073eda7b4edff8342153e24d0de645ea62f2922e8e23 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:36.745 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:48:37.087 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:36.907Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000181921s 2024-04-26T04:48:37.088 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T04:48:37.088 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 systemd[1]: Stopped Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:37.088 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.152s CPU time. 2024-04-26T04:48:37.088 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 systemd[1]: Starting Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:37.088 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 podman[82399]: 2024-04-26 04:48:36.880047324 +0000 UTC m=+0.018125963 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:48:37.088 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:36 smithi005 podman[82399]: 2024-04-26 04:48:36.98029627 +0000 UTC m=+0.118374952 container create c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:37.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-mon[26371]: osdmap e179: 8 total, 8 up, 8 in 2024-04-26T04:48:37.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-mon[26371]: osdmap e180: 8 total, 8 up, 8 in 2024-04-26T04:48:37.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-mon[36961]: osdmap e179: 8 total, 8 up, 8 in 2024-04-26T04:48:37.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-mon[36961]: osdmap e180: 8 total, 8 up, 8 in 2024-04-26T04:48:37.481 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 podman[82399]: 2024-04-26 04:48:37.207790799 +0000 UTC m=+0.345869425 container init c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:37.481 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 podman[82399]: 2024-04-26 04:48:37.210827673 +0000 UTC m=+0.348906305 container start c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:37.481 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.214Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:48:37.481 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.214Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.215Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.215Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.215Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:48:37.482 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:48:37.483 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:48:37.484 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a[82413]: ts=2024-04-26T04:48:37.216Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 bash[82399]: c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa 2024-04-26T04:48:37.485 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 04:48:37 smithi005 systemd[1]: Started Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:37.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:37 smithi111 ceph-mon[30255]: osdmap e179: 8 total, 8 up, 8 in 2024-04-26T04:48:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:37 smithi111 ceph-mon[30255]: osdmap e180: 8 total, 8 up, 8 in 2024-04-26T04:48:38.144 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:48:38.150 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:48:38.151 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:48:38.161 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:48:38.185 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:48:38.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[26371]: pgmap v48: 137 pgs: 137 active+clean; 521 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:48:38.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[26371]: osdmap e181: 8 total, 8 up, 8 in 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[36961]: pgmap v48: 137 pgs: 137 active+clean; 521 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:38 smithi005 ceph-mon[36961]: osdmap e181: 8 total, 8 up, 8 in 2024-04-26T04:48:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:38 smithi111 ceph-mon[30255]: pgmap v48: 137 pgs: 137 active+clean; 521 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:48:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:38 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:38 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:38 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:38 smithi111 ceph-mon[30255]: osdmap e181: 8 total, 8 up, 8 in 2024-04-26T04:48:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:39 smithi111 ceph-mon[30255]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:48:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:39 smithi111 ceph-mon[30255]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:48:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:39 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3721520893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:39 smithi111 ceph-mon[30255]: osdmap e182: 8 total, 8 up, 8 in 2024-04-26T04:48:39.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[36961]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[36961]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3721520893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[36961]: osdmap e182: 8 total, 8 up, 8 in 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[26371]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[26371]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3721520893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:39 smithi005 ceph-mon[26371]: osdmap e182: 8 total, 8 up, 8 in 2024-04-26T04:48:40.156 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:40.180 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:48:40.190 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:48:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:40 smithi111 ceph-mon[30255]: pgmap v51: 137 pgs: 137 active+clean; 541 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.1 MiB/s wr, 178 op/s 2024-04-26T04:48:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:40 smithi111 ceph-mon[30255]: osdmap e183: 8 total, 8 up, 8 in 2024-04-26T04:48:40.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:40 smithi005 ceph-mon[26371]: pgmap v51: 137 pgs: 137 active+clean; 541 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.1 MiB/s wr, 178 op/s 2024-04-26T04:48:40.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:40 smithi005 ceph-mon[26371]: osdmap e183: 8 total, 8 up, 8 in 2024-04-26T04:48:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:40 smithi005 ceph-mon[36961]: pgmap v51: 137 pgs: 137 active+clean; 541 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.1 MiB/s wr, 178 op/s 2024-04-26T04:48:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:40 smithi005 ceph-mon[36961]: osdmap e183: 8 total, 8 up, 8 in 2024-04-26T04:48:41.154 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:40 smithi111 systemd[1]: Stopping Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:41.455 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4261901295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:41.455 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:48:41.455 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:48:41.455 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:48:41.455 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[60876]: ts=2024-04-26T04:48:41.154Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:48:41.456 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 podman[71669]: 2024-04-26 04:48:41.164292957 +0000 UTC m=+0.111420388 container died f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:41.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:41 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4261901295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:41.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:41 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4261901295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:41.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 podman[71669]: 2024-04-26 04:48:41.665236773 +0000 UTC m=+0.612364245 container cleanup f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:41.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 bash[71669]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:48:42.223 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:41 smithi111 podman[71683]: 2024-04-26 04:48:41.849755353 +0000 UTC m=+0.688402310 container remove f0899160bb4901532bf2d1644187183acd16752d77fdd3425e5d33c97ad21a24 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:42.223 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:48:42.224 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 systemd[1]: Stopped Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:42.224 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.622s CPU time. 2024-04-26T04:48:42.224 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 systemd[1]: Starting Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:42.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:42 smithi005 ceph-mon[26371]: pgmap v53: 137 pgs: 137 active+clean; 541 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 7.9 MiB/s wr, 175 op/s 2024-04-26T04:48:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:42 smithi005 ceph-mon[26371]: osdmap e184: 8 total, 8 up, 8 in 2024-04-26T04:48:42.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:42 smithi005 ceph-mon[26371]: osdmap e185: 8 total, 8 up, 8 in 2024-04-26T04:48:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:42 smithi005 ceph-mon[36961]: pgmap v53: 137 pgs: 137 active+clean; 541 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 7.9 MiB/s wr, 175 op/s 2024-04-26T04:48:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:42 smithi005 ceph-mon[36961]: osdmap e184: 8 total, 8 up, 8 in 2024-04-26T04:48:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:42 smithi005 ceph-mon[36961]: osdmap e185: 8 total, 8 up, 8 in 2024-04-26T04:48:42.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-mon[30255]: pgmap v53: 137 pgs: 137 active+clean; 541 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 7.9 MiB/s wr, 175 op/s 2024-04-26T04:48:42.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-mon[30255]: osdmap e184: 8 total, 8 up, 8 in 2024-04-26T04:48:42.487 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-mon[30255]: osdmap e185: 8 total, 8 up, 8 in 2024-04-26T04:48:42.488 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 podman[71783]: 2024-04-26 04:48:42.211514182 +0000 UTC m=+0.019707640 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:48:42.488 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 podman[71783]: 2024-04-26 04:48:42.367631411 +0000 UTC m=+0.175824842 container create cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 podman[71783]: 2024-04-26 04:48:42.553220198 +0000 UTC m=+0.361413639 container init cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 podman[71783]: 2024-04-26 04:48:42.55587573 +0000 UTC m=+0.364069172 container start cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.579Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.579Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.579Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi111 (none))" 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.579Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.579Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.580Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.581Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:48:42.769 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.582Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.582Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.584Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.584Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.639µs 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.584Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.584Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.598Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.598Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.598Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.797µs wal_replay_duration=14.078639ms wbl_replay_duration=93ns total_replay_duration=14.118744ms 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.601Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.601Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.601Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 bash[71783]: cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 2024-04-26T04:48:42.770 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 systemd[1]: Started Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:42.771 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.620Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.042221ms db_storage=1.13µs remote_storage=1.155µs web_handler=461ns query_engine=714ns scrape=365.016µs scrape_sd=108.454µs notify=9.627µs notify_sd=8.384µs rules=18.322568ms tracing=6.908µs 2024-04-26T04:48:42.771 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.620Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:48:42.771 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:42.620Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:48:43.561 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 0 2024-04-26T04:48:43.637 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:48:43.753 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:48:43.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: Reconfiguring daemon grafana.a on smithi111 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: osdmap e186: 8 total, 8 up, 8 in 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: pgmap v57: 137 pgs: 137 active+clean; 561 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 4.0 MiB/s wr, 176 op/s 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3265922268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:43.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:43.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:48:43.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: Reconfiguring daemon grafana.a on smithi111 2024-04-26T04:48:43.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: osdmap e186: 8 total, 8 up, 8 in 2024-04-26T04:48:43.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: pgmap v57: 137 pgs: 137 active+clean; 561 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 4.0 MiB/s wr, 176 op/s 2024-04-26T04:48:43.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:43 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3265922268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:44.012 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:44.012 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:44.012 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:48:44.012 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: Reconfiguring daemon grafana.a on smithi111 2024-04-26T04:48:44.013 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: osdmap e186: 8 total, 8 up, 8 in 2024-04-26T04:48:44.013 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: pgmap v57: 137 pgs: 137 active+clean; 561 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 4.0 MiB/s wr, 176 op/s 2024-04-26T04:48:44.013 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:43 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3265922268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:45.229 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 04:48:44 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T04:48:44.911Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003567447s 2024-04-26T04:48:45.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:45 smithi005 ceph-mon[26371]: osdmap e187: 8 total, 8 up, 8 in 2024-04-26T04:48:45.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:45 smithi005 ceph-mon[26371]: pgmap v59: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 4.0 MiB/s wr, 176 op/s 2024-04-26T04:48:45.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:45 smithi005 ceph-mon[36961]: osdmap e187: 8 total, 8 up, 8 in 2024-04-26T04:48:45.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:45 smithi005 ceph-mon[36961]: pgmap v59: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 4.0 MiB/s wr, 176 op/s 2024-04-26T04:48:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:45 smithi111 ceph-mon[30255]: osdmap e187: 8 total, 8 up, 8 in 2024-04-26T04:48:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:45 smithi111 ceph-mon[30255]: pgmap v59: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s rd, 4.0 MiB/s wr, 176 op/s 2024-04-26T04:48:46.893 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (12427 ms) 2024-04-26T04:48:46.894 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-04-26T04:48:46.922 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:46.933 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:48:46.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:46 smithi005 ceph-mon[26371]: osdmap e188: 8 total, 8 up, 8 in 2024-04-26T04:48:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:46 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3958888621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:46 smithi005 ceph-mon[36961]: osdmap e188: 8 total, 8 up, 8 in 2024-04-26T04:48:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:46 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3958888621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:46.985 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:48:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:46 smithi111 ceph-mon[30255]: osdmap e188: 8 total, 8 up, 8 in 2024-04-26T04:48:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:46 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3958888621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:47.915 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:47 smithi111 ceph-mon[30255]: osdmap e189: 8 total, 8 up, 8 in 2024-04-26T04:48:47.915 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:47 smithi111 ceph-mon[30255]: pgmap v62: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 174 op/s 2024-04-26T04:48:47.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:47 smithi005 ceph-mon[26371]: osdmap e189: 8 total, 8 up, 8 in 2024-04-26T04:48:47.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:47 smithi005 ceph-mon[26371]: pgmap v62: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 174 op/s 2024-04-26T04:48:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:47 smithi005 ceph-mon[36961]: osdmap e189: 8 total, 8 up, 8 in 2024-04-26T04:48:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:47 smithi005 ceph-mon[36961]: pgmap v62: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 174 op/s 2024-04-26T04:48:48.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:47] "GET /metrics HTTP/1.1" 200 38773 "" "Prometheus/2.43.0" 2024-04-26T04:48:48.678 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:48:48.698 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:48:48.698 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:48:48.698 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:48:48.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:48 smithi005 ceph-mon[26371]: osdmap e190: 8 total, 8 up, 8 in 2024-04-26T04:48:48.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:48 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3048933741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:48 smithi005 ceph-mon[36961]: osdmap e190: 8 total, 8 up, 8 in 2024-04-26T04:48:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:48 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3048933741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:48 smithi111 ceph-mon[30255]: osdmap e190: 8 total, 8 up, 8 in 2024-04-26T04:48:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:48 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3048933741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:49.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:48 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:49 smithi005 ceph-mon[26371]: osdmap e191: 8 total, 8 up, 8 in 2024-04-26T04:48:49.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:49 smithi005 ceph-mon[26371]: pgmap v65: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 480 op/s 2024-04-26T04:48:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:49 smithi005 ceph-mon[36961]: osdmap e191: 8 total, 8 up, 8 in 2024-04-26T04:48:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:49 smithi005 ceph-mon[36961]: pgmap v65: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 480 op/s 2024-04-26T04:48:50.012 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:49 smithi111 ceph-mon[30255]: osdmap e191: 8 total, 8 up, 8 in 2024-04-26T04:48:50.013 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:49 smithi111 ceph-mon[30255]: pgmap v65: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 480 op/s 2024-04-26T04:48:50.689 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:48:50.695 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:48:50.696 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:48:50.705 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:48:50.730 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:48:50.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:50 smithi005 ceph-mon[26371]: osdmap e192: 8 total, 8 up, 8 in 2024-04-26T04:48:50.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:50 smithi005 ceph-mon[36961]: osdmap e192: 8 total, 8 up, 8 in 2024-04-26T04:48:51.011 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:50 smithi111 ceph-mon[30255]: osdmap e192: 8 total, 8 up, 8 in 2024-04-26T04:48:51.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:51 smithi005 ceph-mon[36961]: osdmap e193: 8 total, 8 up, 8 in 2024-04-26T04:48:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:51 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/855950255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:51 smithi005 ceph-mon[36961]: pgmap v68: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 480 op/s 2024-04-26T04:48:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:51 smithi005 ceph-mon[26371]: osdmap e193: 8 total, 8 up, 8 in 2024-04-26T04:48:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:51 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/855950255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:51 smithi005 ceph-mon[26371]: pgmap v68: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 480 op/s 2024-04-26T04:48:52.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:51 smithi111 ceph-mon[30255]: osdmap e193: 8 total, 8 up, 8 in 2024-04-26T04:48:52.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:51 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/855950255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:52.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:51 smithi111 ceph-mon[30255]: pgmap v68: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 480 op/s 2024-04-26T04:48:52.149 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:52.173 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:48:52.182 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:48:52.686 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:52 smithi111 ceph-mon[30255]: osdmap e194: 8 total, 8 up, 8 in 2024-04-26T04:48:52.686 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:52 smithi111 systemd[1]: Stopping Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:52 smithi005 ceph-mon[26371]: osdmap e194: 8 total, 8 up, 8 in 2024-04-26T04:48:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:52 smithi005 ceph-mon[26371]: osdmap e195: 8 total, 8 up, 8 in 2024-04-26T04:48:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:52 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:52 smithi005 ceph-mon[36961]: osdmap e194: 8 total, 8 up, 8 in 2024-04-26T04:48:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:52 smithi005 ceph-mon[36961]: osdmap e195: 8 total, 8 up, 8 in 2024-04-26T04:48:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:52 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:52 smithi111 ceph-mon[30255]: osdmap e195: 8 total, 8 up, 8 in 2024-04-26T04:48:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:52 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:53.019 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:52 smithi111 podman[72126]: 2024-04-26 04:48:52.75753477 +0000 UTC m=+0.125167213 container died d26b90f67e2262cf37b21e33dc5a9949d089f23da44130428a34933c7fc00a02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, name=ubi8, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, version=8.5, com.redhat.component=ubi8-container, release=239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:48:53.458 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 podman[72126]: 2024-04-26 04:48:53.256034598 +0000 UTC m=+0.623667073 container cleanup d26b90f67e2262cf37b21e33dc5a9949d089f23da44130428a34933c7fc00a02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, version=8.5, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vendor=Red Hat, Inc., vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:48:53.458 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 bash[72126]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a 2024-04-26T04:48:53.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 podman[72140]: 2024-04-26 04:48:53.457186976 +0000 UTC m=+0.697670562 container remove d26b90f67e2262cf37b21e33dc5a9949d089f23da44130428a34933c7fc00a02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, name=ubi8, maintainer=Paul Cuzner , description=Ceph Grafana Container, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.openshift.expose-services=, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:48:53.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:48:53.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:48:53.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:53.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.199s CPU time. 2024-04-26T04:48:53.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:54.069 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 podman[72234]: 2024-04-26 04:48:53.716395104 +0000 UTC m=+0.017365855 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:48:54.069 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:53 smithi111 podman[72234]: 2024-04-26 04:48:53.858250744 +0000 UTC m=+0.159221483 container create 88e686d28d157f26ed0250bd6f4b799641b93f5a8f6293720053f57b7a48d1f8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, release=239.1651231664, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:48:54.069 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:54 smithi111 podman[72234]: 2024-04-26 04:48:54.069014636 +0000 UTC m=+0.369985379 container init 88e686d28d157f26ed0250bd6f4b799641b93f5a8f6293720053f57b7a48d1f8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, description=Ceph Grafana Container, release=239.1651231664, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:48:54.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:54 smithi111 ceph-mon[30255]: osdmap e196: 8 total, 8 up, 8 in 2024-04-26T04:48:54.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:54 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2718466370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:54.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:54 smithi111 ceph-mon[30255]: pgmap v72: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 161 KiB/s rd, 9.1 MiB/s wr, 232 op/s 2024-04-26T04:48:54.462 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:54 smithi111 podman[72234]: 2024-04-26 04:48:54.071782027 +0000 UTC m=+0.372752768 container start 88e686d28d157f26ed0250bd6f4b799641b93f5a8f6293720053f57b7a48d1f8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.buildah.version=1.32.0, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, version=8.5, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.openshift.expose-services=, description=Ceph Grafana Container, release=239.1651231664, com.redhat.component=ubi8-container, architecture=x86_64) 2024-04-26T04:48:54.462 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:54 smithi111 bash[72234]: 88e686d28d157f26ed0250bd6f4b799641b93f5a8f6293720053f57b7a48d1f8 2024-04-26T04:48:54.462 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:54 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:54.463 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[72248]: logger=settings t=2024-04-26T04:48:54.228205594Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:48:54.463 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:54 smithi111 podman[72294]: 2024-04-26 04:48:54.258298864 +0000 UTC m=+0.019178552 container died 88e686d28d157f26ed0250bd6f4b799641b93f5a8f6293720053f57b7a48d1f8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , name=ubi8, version=8.5, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:48:54.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:54 smithi005 ceph-mon[26371]: osdmap e196: 8 total, 8 up, 8 in 2024-04-26T04:48:54.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:54 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2718466370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:54.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:54 smithi005 ceph-mon[26371]: pgmap v72: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 161 KiB/s rd, 9.1 MiB/s wr, 232 op/s 2024-04-26T04:48:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:54 smithi005 ceph-mon[36961]: osdmap e196: 8 total, 8 up, 8 in 2024-04-26T04:48:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:54 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2718466370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:54.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:54 smithi005 ceph-mon[36961]: pgmap v72: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 161 KiB/s rd, 9.1 MiB/s wr, 232 op/s 2024-04-26T04:48:55.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[36961]: osdmap e197: 8 total, 8 up, 8 in 2024-04-26T04:48:55.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[36961]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[36961]: Reconfiguring daemon node-exporter.b on smithi111 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[26371]: osdmap e197: 8 total, 8 up, 8 in 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[26371]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:48:55.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:55 smithi005 ceph-mon[26371]: Reconfiguring daemon node-exporter.b on smithi111 2024-04-26T04:48:55.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:55 smithi111 ceph-mon[30255]: osdmap e197: 8 total, 8 up, 8 in 2024-04-26T04:48:55.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:55 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:55.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:55 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:48:55.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:55 smithi111 ceph-mon[30255]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:48:55.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:55 smithi111 ceph-mon[30255]: Reconfiguring daemon node-exporter.b on smithi111 2024-04-26T04:48:55.567 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:48:55.577 INFO:tasks.workunit.client.0.smithi005.stdout:write: * 83886080~1000 2024-04-26T04:48:55.606 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:48:55.744 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:48:56.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:56 smithi005 ceph-mon[26371]: osdmap e198: 8 total, 8 up, 8 in 2024-04-26T04:48:56.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:56 smithi005 ceph-mon[26371]: pgmap v75: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 162 KiB/s rd, 9.2 MiB/s wr, 232 op/s 2024-04-26T04:48:56.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:56 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3715798247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:56 smithi005 ceph-mon[36961]: osdmap e198: 8 total, 8 up, 8 in 2024-04-26T04:48:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:56 smithi005 ceph-mon[36961]: pgmap v75: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 162 KiB/s rd, 9.2 MiB/s wr, 232 op/s 2024-04-26T04:48:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:56 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3715798247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:56.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:56 smithi111 ceph-mon[30255]: osdmap e198: 8 total, 8 up, 8 in 2024-04-26T04:48:56.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:56 smithi111 ceph-mon[30255]: pgmap v75: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 162 KiB/s rd, 9.2 MiB/s wr, 232 op/s 2024-04-26T04:48:56.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:56 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3715798247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:57.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:57 smithi005 ceph-mon[26371]: osdmap e199: 8 total, 8 up, 8 in 2024-04-26T04:48:57.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:57 smithi005 ceph-mon[36961]: osdmap e199: 8 total, 8 up, 8 in 2024-04-26T04:48:57.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:57 smithi111 ceph-mon[30255]: osdmap e199: 8 total, 8 up, 8 in 2024-04-26T04:48:58.175 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:58 smithi111 ceph-mon[30255]: osdmap e200: 8 total, 8 up, 8 in 2024-04-26T04:48:58.175 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:58 smithi111 ceph-mon[30255]: pgmap v78: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:48:58.175 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:58 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2412891830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:58.175 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:48:57] "GET /metrics HTTP/1.1" 200 38775 "" "Prometheus/2.43.0" 2024-04-26T04:48:58.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:58 smithi005 ceph-mon[36961]: osdmap e200: 8 total, 8 up, 8 in 2024-04-26T04:48:58.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:58 smithi005 ceph-mon[36961]: pgmap v78: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:48:58.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:58 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2412891830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:58 smithi005 ceph-mon[26371]: osdmap e200: 8 total, 8 up, 8 in 2024-04-26T04:48:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:58 smithi005 ceph-mon[26371]: pgmap v78: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:48:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:58 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2412891830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:58.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:58 smithi111 systemd[1]: Stopping Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:58.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:58 smithi111 podman[72294]: 2024-04-26 04:48:58.17504551 +0000 UTC m=+3.935925204 container remove 88e686d28d157f26ed0250bd6f4b799641b93f5a8f6293720053f57b7a48d1f8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, version=8.5, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vendor=Red Hat, Inc., io.buildah.version=1.32.0, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.component=ubi8-container, vcs-type=git, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:48:58.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:58 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:48:58.527 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (11635 ms) 2024-04-26T04:48:58.527 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-04-26T04:48:58.556 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:48:58.568 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 41943040~10 2024-04-26T04:48:58.578 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:48:58.863 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:58 smithi111 podman[72422]: 2024-04-26 04:48:58.572006629 +0000 UTC m=+0.157754934 container died 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:48:58.863 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:48:58 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:48:58.864 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:48:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:48:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:59.163 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:58 smithi111 podman[72422]: 2024-04-26 04:48:58.953343352 +0000 UTC m=+0.539091644 container cleanup 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:48:59.164 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:58 smithi111 bash[72422]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T04:48:59.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:48:59 smithi005 ceph-mon[26371]: osdmap e201: 8 total, 8 up, 8 in 2024-04-26T04:48:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:48:59 smithi005 ceph-mon[36961]: osdmap e201: 8 total, 8 up, 8 in 2024-04-26T04:48:59.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-mon[30255]: osdmap e201: 8 total, 8 up, 8 in 2024-04-26T04:48:59.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 podman[72436]: 2024-04-26 04:48:59.205148773 +0000 UTC m=+0.636622870 container remove 2e3c3a24be0dd59ba9b21820fa33f4a7b06552c7a70503bc9c563c3a9a42aa03 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:48:59.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:48:59.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T04:48:59.518 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 systemd[1]: Stopped Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:59.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.078s CPU time. 2024-04-26T04:48:59.519 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 systemd[1]: Starting Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:59.825 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 podman[72530]: 2024-04-26 04:48:59.420390433 +0000 UTC m=+0.018062978 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:48:59.825 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 podman[72530]: 2024-04-26 04:48:59.530928972 +0000 UTC m=+0.128601493 container create cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:48:59.825 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 podman[72530]: 2024-04-26 04:48:59.741543098 +0000 UTC m=+0.339215619 container init cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:48:59.825 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 podman[72530]: 2024-04-26 04:48:59.744079121 +0000 UTC m=+0.341751643 container start cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:48:59.825 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.747Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.747Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.747Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.747Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:48:59.826 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:48:59.827 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:48:59.828 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:48:59.829 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.748Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.749Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b[72545]: ts=2024-04-26T04:48:59.749Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:48:59.830 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 bash[72530]: cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 2024-04-26T04:49:00.118 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 04:48:59 smithi111 systemd[1]: Started Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:00.118 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:59] ENGINE Bus STOPPING 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:59] ENGINE Bus STOPPED 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:48:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:48:59] ENGINE Bus STARTING 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Serving on http://:::9283 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STARTED 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STOPPING 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STOPPED 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STARTING 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Serving on http://:::9283 2024-04-26T04:49:00.119 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STARTED 2024-04-26T04:49:00.186 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:49:00.204 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:00.388 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STOPPING 2024-04-26T04:49:00.388 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:49:00.388 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STOPPED 2024-04-26T04:49:00.388 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STARTING 2024-04-26T04:49:00.388 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Serving on http://:::9283 2024-04-26T04:49:00.388 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:00] ENGINE Bus STARTED 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: osdmap e202: 8 total, 8 up, 8 in 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: pgmap v81: 137 pgs: 137 active+clean; 617 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 442 op/s 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:00.389 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111:9095"}]: dispatch 2024-04-26T04:49:00.390 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.390 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:49:00.390 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111:3000"}]: dispatch 2024-04-26T04:49:00.390 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.390 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:00 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: osdmap e202: 8 total, 8 up, 8 in 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: pgmap v81: 137 pgs: 137 active+clean; 617 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 442 op/s 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111:9095"}]: dispatch 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:49:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111:3000"}]: dispatch 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: osdmap e202: 8 total, 8 up, 8 in 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: pgmap v81: 137 pgs: 137 active+clean; 617 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 442 op/s 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111:9095"}]: dispatch 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:49:00.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111:3000"}]: dispatch 2024-04-26T04:49:00.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:00.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:00 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:01.325 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:01.335 INFO:tasks.workunit.client.0.smithi005.stdout:write: Y 41943040~5 2024-04-26T04:49:01.443 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:01.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111:9095"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111:3000"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: osdmap e203: 8 total, 8 up, 8 in 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/698910797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:01.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111:9095"}]: dispatch 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111:3000"}]: dispatch 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: osdmap e203: 8 total, 8 up, 8 in 2024-04-26T04:49:01.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:01 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/698910797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:01.517 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi005.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111:9095"}]: dispatch 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi111:3000"}]: dispatch 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: osdmap e203: 8 total, 8 up, 8 in 2024-04-26T04:49:01.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:01 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/698910797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:02.227 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (3699 ms) 2024-04-26T04:49:02.227 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-04-26T04:49:02.256 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:02.266 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:49:02.328 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:49:02.444 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: osdmap e204: 8 total, 8 up, 8 in 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: pgmap v84: 137 pgs: 137 active+clean; 617 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 442 op/s 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[26371]: osdmap e205: 8 total, 8 up, 8 in 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: osdmap e204: 8 total, 8 up, 8 in 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: pgmap v84: 137 pgs: 137 active+clean; 617 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 442 op/s 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:02.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:02.446 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:02.446 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:02.446 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:02 smithi005 ceph-mon[36961]: osdmap e205: 8 total, 8 up, 8 in 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: osdmap e204: 8 total, 8 up, 8 in 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: pgmap v84: 137 pgs: 137 active+clean; 617 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 442 op/s 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:02 smithi111 ceph-mon[30255]: osdmap e205: 8 total, 8 up, 8 in 2024-04-26T04:49:03.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:03 smithi005 ceph-mon[26371]: Upgrade: Updating mgr.y 2024-04-26T04:49:03.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:03 smithi005 ceph-mon[26371]: Deploying daemon mgr.y on smithi005 2024-04-26T04:49:03.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:03 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2225957627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:03.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:03 smithi005 ceph-mon[36961]: Upgrade: Updating mgr.y 2024-04-26T04:49:03.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:03 smithi005 ceph-mon[36961]: Deploying daemon mgr.y on smithi005 2024-04-26T04:49:03.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:03 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2225957627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:03.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:03 smithi111 ceph-mon[30255]: Upgrade: Updating mgr.y 2024-04-26T04:49:03.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:03 smithi111 ceph-mon[30255]: Deploying daemon mgr.y on smithi005 2024-04-26T04:49:03.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:03 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2225957627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:03.784 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:03 smithi005 systemd[1]: Stopping Ceph mgr.y for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:04.046 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:03 smithi005 podman[83009]: 2024-04-26 04:49:03.783703946 +0000 UTC m=+0.135663081 container died 4e6b89d741aa19e2eb4a1f03cd89029126ed0e146ce7f7872372cc2d3e52de43 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:49:04.211 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:49:04.228 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:49:04.228 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:49:04.228 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:49:04.429 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:04 smithi005 ceph-mon[26371]: osdmap e206: 8 total, 8 up, 8 in 2024-04-26T04:49:04.429 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:04 smithi005 ceph-mon[26371]: pgmap v87: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 4.0 MiB/s wr, 457 op/s 2024-04-26T04:49:04.430 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:04 smithi005 ceph-mon[36961]: osdmap e206: 8 total, 8 up, 8 in 2024-04-26T04:49:04.430 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:04 smithi005 ceph-mon[36961]: pgmap v87: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 4.0 MiB/s wr, 457 op/s 2024-04-26T04:49:04.430 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 podman[83009]: 2024-04-26 04:49:04.139185637 +0000 UTC m=+0.491144771 container cleanup 4e6b89d741aa19e2eb4a1f03cd89029126ed0e146ce7f7872372cc2d3e52de43 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:49:04.430 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 bash[83009]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y 2024-04-26T04:49:04.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:04 smithi111 ceph-mon[30255]: osdmap e206: 8 total, 8 up, 8 in 2024-04-26T04:49:04.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:04 smithi111 ceph-mon[30255]: pgmap v87: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 4.0 MiB/s wr, 457 op/s 2024-04-26T04:49:04.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 podman[83023]: 2024-04-26 04:49:04.429345115 +0000 UTC m=+0.646035806 container remove 4e6b89d741aa19e2eb4a1f03cd89029126ed0e146ce7f7872372cc2d3e52de43 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:04.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:49:04.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-26T04:49:04.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 systemd[1]: Stopped Ceph mgr.y for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:04.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 1.681s CPU time. 2024-04-26T04:49:05.147 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:04 smithi005 systemd[1]: Starting Ceph mgr.y for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:05.147 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 podman[83156]: 2024-04-26 04:49:04.928898947 +0000 UTC m=+0.019533716 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:05.147 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 podman[83156]: 2024-04-26 04:49:05.038944733 +0000 UTC m=+0.129579500 container create c895550b1a50d0276896cedbe4f9108819594711d6b60a0372e8cf1d715c8f77 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-26T04:49:05.422 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:05 smithi005 ceph-mon[26371]: osdmap e207: 8 total, 8 up, 8 in 2024-04-26T04:49:05.423 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:05 smithi005 ceph-mon[36961]: osdmap e207: 8 total, 8 up, 8 in 2024-04-26T04:49:05.423 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 podman[83156]: 2024-04-26 04:49:05.25858218 +0000 UTC m=+0.349216948 container init c895550b1a50d0276896cedbe4f9108819594711d6b60a0372e8cf1d715c8f77 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb) 2024-04-26T04:49:05.423 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 podman[83156]: 2024-04-26 04:49:05.261430845 +0000 UTC m=+0.352065621 container start c895550b1a50d0276896cedbe4f9108819594711d6b60a0372e8cf1d715c8f77 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T04:49:05.423 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 ceph-mgr[83194]: -- 172.21.15.5:0/3372264479 <== mon.2 v2:172.21.15.5:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562d65da05a0 con 0x562d65d74c00 2024-04-26T04:49:05.423 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 bash[83156]: c895550b1a50d0276896cedbe4f9108819594711d6b60a0372e8cf1d715c8f77 2024-04-26T04:49:05.423 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 systemd[1]: Started Ceph mgr.y for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:05 smithi111 ceph-mon[30255]: osdmap e207: 8 total, 8 up, 8 in 2024-04-26T04:49:05.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:05.421+0000 7fb1ef61f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:05.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:05.617+0000 7fb1ef61f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:06.081 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:05.729+0000 7fb1ef61f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:06.081 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:06.079+0000 7fb1ef61f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:06.216 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:49:06.224 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:49:06.224 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:49:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:49:06.248 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:49:06.381 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:06.169+0000 7fb1ef61f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:06.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[26371]: pgmap v89: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 4.0 MiB/s wr, 455 op/s 2024-04-26T04:49:06.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[26371]: osdmap e208: 8 total, 8 up, 8 in 2024-04-26T04:49:06.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/781729289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:06.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:06.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:06.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:06.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[36961]: pgmap v89: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 4.0 MiB/s wr, 455 op/s 2024-04-26T04:49:06.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[36961]: osdmap e208: 8 total, 8 up, 8 in 2024-04-26T04:49:06.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/781729289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:06.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:06.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:06.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:06 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:06.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:06 smithi111 ceph-mon[30255]: pgmap v89: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 4.0 MiB/s wr, 455 op/s 2024-04-26T04:49:06.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:06 smithi111 ceph-mon[30255]: osdmap e208: 8 total, 8 up, 8 in 2024-04-26T04:49:06.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:06 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/781729289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:06.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:06 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:06.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:06 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:06.462 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:06 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:07.367 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-mon[36961]: osdmap e209: 8 total, 8 up, 8 in 2024-04-26T04:49:07.367 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-mon[36961]: osdmap e210: 8 total, 8 up, 8 in 2024-04-26T04:49:07.367 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-mon[26371]: osdmap e209: 8 total, 8 up, 8 in 2024-04-26T04:49:07.367 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-mon[26371]: osdmap e210: 8 total, 8 up, 8 in 2024-04-26T04:49:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:07 smithi111 ceph-mon[30255]: osdmap e209: 8 total, 8 up, 8 in 2024-04-26T04:49:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:07 smithi111 ceph-mon[30255]: osdmap e210: 8 total, 8 up, 8 in 2024-04-26T04:49:07.717 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:07.522+0000 7fb1ef61f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:07.717 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:07.715+0000 7fb1ef61f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:07.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:07.930+0000 7fb1ef61f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:08.151 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:08.173 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:49:08.182 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:08.212 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:49:07] "GET /metrics HTTP/1.1" 200 38769 "" "Prometheus/2.43.0" 2024-04-26T04:49:08.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[26371]: pgmap v93: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2717212825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[26371]: osdmap e211: 8 total, 8 up, 8 in 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:08.263+0000 7fb1ef61f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:08.362+0000 7fb1ef61f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[36961]: pgmap v93: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2717212825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:08.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-mon[36961]: osdmap e211: 8 total, 8 up, 8 in 2024-04-26T04:49:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:08 smithi111 ceph-mon[30255]: pgmap v93: 137 pgs: 137 active+clean; 633 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:08 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:08 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2717212825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:08 smithi111 ceph-mon[30255]: osdmap e211: 8 total, 8 up, 8 in 2024-04-26T04:49:08.956 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:49:08.957 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:08 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-26T04:49:08.957 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:08 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:08.957 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:08 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:08.959 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:08.575+0000 7fb1ef61f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:08.959 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:08 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:08.794+0000 7fb1ef61f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:09.652 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 podman[73153]: 2024-04-26 04:49:09.282601468 +0000 UTC m=+0.019415673 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:49:09.653 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 podman[73153]: 2024-04-26 04:49:09.416635819 +0000 UTC m=+0.153450049 container create 6cae217d6ea651f3328b98ad1645984a108160e5929492f57a134cc14ce65309 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., version=8.5, release=239.1651231664, io.buildah.version=1.32.0, distribution-scope=public, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, vcs-type=git, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=) 2024-04-26T04:49:09.653 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 podman[73153]: 2024-04-26 04:49:09.652411347 +0000 UTC m=+0.389225540 container init 6cae217d6ea651f3328b98ad1645984a108160e5929492f57a134cc14ce65309 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, version=8.5, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner ) 2024-04-26T04:49:09.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:09 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:09.448+0000 7fb1ef61f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:10.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 podman[73153]: 2024-04-26 04:49:09.655127078 +0000 UTC m=+0.391941267 container start 6cae217d6ea651f3328b98ad1645984a108160e5929492f57a134cc14ce65309 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, version=8.5, release=239.1651231664, maintainer=Paul Cuzner , io.buildah.version=1.32.0, vcs-type=git, distribution-scope=public, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container) 2024-04-26T04:49:10.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 bash[73153]: 6cae217d6ea651f3328b98ad1645984a108160e5929492f57a134cc14ce65309 2024-04-26T04:49:10.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:10.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[73167]: logger=settings t=2024-04-26T04:49:09.803867467Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:49:10.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:09 smithi111 podman[73184]: 2024-04-26 04:49:09.829537125 +0000 UTC m=+0.018330420 container died 6cae217d6ea651f3328b98ad1645984a108160e5929492f57a134cc14ce65309 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, version=8.5, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , name=ubi8) 2024-04-26T04:49:10.223 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:09 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:09.914+0000 7fb1ef61f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:10.223 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:10.020+0000 7fb1ef61f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:10.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:10.246+0000 7fb1ef61f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: pgmap v95: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 112 KiB/s rd, 8.0 MiB/s wr, 163 op/s 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: osdmap e212: 8 total, 8 up, 8 in 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2592253950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: pgmap v95: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 112 KiB/s rd, 8.0 MiB/s wr, 163 op/s 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: osdmap e212: 8 total, 8 up, 8 in 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2592253950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: pgmap v95: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 112 KiB/s rd, 8.0 MiB/s wr, 163 op/s 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: osdmap e212: 8 total, 8 up, 8 in 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.481 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.482 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:10.482 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:10 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2592253950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:10.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:10 smithi111 podman[73184]: 2024-04-26 04:49:10.481610483 +0000 UTC m=+0.670403819 container remove 6cae217d6ea651f3328b98ad1645984a108160e5929492f57a134cc14ce65309 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vcs-type=git, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5) 2024-04-26T04:49:10.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:10 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:49:10.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:10 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:49:11.186 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:10.812+0000 7fb1ef61f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:11.187 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:10.914+0000 7fb1ef61f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:11.462 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-mon[26371]: osdmap e213: 8 total, 8 up, 8 in 2024-04-26T04:49:11.462 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:11.462 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:11.463 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-mon[36961]: osdmap e213: 8 total, 8 up, 8 in 2024-04-26T04:49:11.463 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:11.463 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:11.463 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:11.185+0000 7fb1ef61f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:11.463 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:11.345+0000 7fb1ef61f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:49:11.463 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:11.461+0000 7fb1ef61f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:49:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:11 smithi111 ceph-mon[30255]: osdmap e213: 8 total, 8 up, 8 in 2024-04-26T04:49:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:11 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:11 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:11.713 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:11.628+0000 7fb1ef61f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:11.818 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: after_prepare_snap 2024-04-26T04:49:11.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:11 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:11.742+0000 7fb1ef61f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:49:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:12 smithi111 ceph-mon[30255]: pgmap v98: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 111 KiB/s rd, 7.9 MiB/s wr, 161 op/s 2024-04-26T04:49:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:12 smithi111 ceph-mon[30255]: osdmap e214: 8 total, 8 up, 8 in 2024-04-26T04:49:12.526 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-mon[36961]: pgmap v98: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 111 KiB/s rd, 7.9 MiB/s wr, 161 op/s 2024-04-26T04:49:12.526 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-mon[36961]: osdmap e214: 8 total, 8 up, 8 in 2024-04-26T04:49:12.526 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-mon[26371]: pgmap v98: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 111 KiB/s rd, 7.9 MiB/s wr, 161 op/s 2024-04-26T04:49:12.527 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-mon[26371]: osdmap e214: 8 total, 8 up, 8 in 2024-04-26T04:49:12.837 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:12.623+0000 7fb1ef61f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:12.856+0000 7fb1ef61f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:12] ENGINE Bus STARTING 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: CherryPy Checker: 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: The Application mounted at '' has an empty config. 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:12] ENGINE Serving on http://:::9283 2024-04-26T04:49:13.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:12 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:12] ENGINE Bus STARTED 2024-04-26T04:49:13.608 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:49:13.638 INFO:tasks.workunit.client.0.smithi005.stdout:write: * 0~1000 2024-04-26T04:49:13.645 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:13.781 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:13.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: osdmap e215: 8 total, 8 up, 8 in 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1046813662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: Standby manager daemon y restarted 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: Standby manager daemon y started 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[26371]: pgmap v101: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 6.7 KiB/s wr, 261 op/s 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: osdmap e215: 8 total, 8 up, 8 in 2024-04-26T04:49:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1046813662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: Standby manager daemon y restarted 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: Standby manager daemon y started 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:13 smithi005 ceph-mon[36961]: pgmap v101: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 6.7 KiB/s wr, 261 op/s 2024-04-26T04:49:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: osdmap e215: 8 total, 8 up, 8 in 2024-04-26T04:49:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1046813662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: Standby manager daemon y restarted 2024-04-26T04:49:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: Standby manager daemon y started 2024-04-26T04:49:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:49:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:49:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:13 smithi111 ceph-mon[30255]: pgmap v101: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 6.7 KiB/s wr, 261 op/s 2024-04-26T04:49:14.762 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: osdmap e216: 8 total, 8 up, 8 in 2024-04-26T04:49:14.762 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: mgrmap e27: x(active, since 66s), standbys: y 2024-04-26T04:49:14.762 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:14.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:14.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.763 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:14] ENGINE Bus STOPPING 2024-04-26T04:49:14.763 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: osdmap e216: 8 total, 8 up, 8 in 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: mgrmap e27: x(active, since 66s), standbys: y 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:14.764 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.917 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: osdmap e216: 8 total, 8 up, 8 in 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: mgrmap e27: x(active, since 66s), standbys: y 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:14.735+0000 7fdbe9ba1700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ignoring --setuser ceph since I am not root 2024-04-26T04:49:14.918 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ignoring --setgroup ceph since I am not root 2024-04-26T04:49:14.919 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-mgr[63687]: -- 172.21.15.111:0/2545433203 <== mon.1 v2:172.21.15.111:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56174b5725a0 con 0x56174b546800 2024-04-26T04:49:15.198 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:14.948+0000 7f24a9cc3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:15.198 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:15.197+0000 7f24a9cc3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:14] ENGINE Bus STOPPED 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:14] ENGINE Bus STARTING 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.887+0000 7fb150210700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.887+0000 7fb150210700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.887+0000 7fb150210700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.887+0000 7fb150210700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.887+0000 7fb150210700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.889+0000 7fb14d20a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.889+0000 7fb14d20a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.889+0000 7fb14d20a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.889+0000 7fb14d20a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:14.889+0000 7fb14d20a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:14] ENGINE Serving on http://:::9283 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:14] ENGINE Bus STARTED 2024-04-26T04:49:15.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:15.306+0000 7f24a9cc3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: osdmap e217: 8 total, 8 up, 8 in 2024-04-26T04:49:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:49:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:49:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: Activating manager daemon y 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: osdmap e218: 8 total, 8 up, 8 in 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.24877 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: mgrmap e28: y(active, starting, since 0.00708955s) 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:49:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: Manager daemon y is now available 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:49:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:49:15.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1135343572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:15.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[26371]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:49:15.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: osdmap e217: 8 total, 8 up, 8 in 2024-04-26T04:49:15.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:15.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:49:15.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: Activating manager daemon y 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: osdmap e218: 8 total, 8 up, 8 in 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.24877 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: mgrmap e28: y(active, starting, since 0.00708955s) 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:49:15.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: Manager daemon y is now available 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1135343572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:15.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:15 smithi005 ceph-mon[36961]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:49:16.013 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:15.626+0000 7f24a9cc3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:16.013 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:15.722+0000 7f24a9cc3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:16.013 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: osdmap e217: 8 total, 8 up, 8 in 2024-04-26T04:49:16.013 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.24877 172.21.15.111:0/3593349548' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: Activating manager daemon y 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: osdmap e218: 8 total, 8 up, 8 in 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.24877 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: mgrmap e28: y(active, starting, since 0.00708955s) 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:16.014 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:49:16.015 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: Manager daemon y is now available 2024-04-26T04:49:16.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:16.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:16.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:49:16.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:49:16.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1135343572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:16.016 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:15 smithi111 ceph-mon[30255]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:49:16.089 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (13862 ms) 2024-04-26T04:49:16.089 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-04-26T04:49:16.121 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:16.133 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:49:16.200 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:49:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: [26/Apr/2024:04:49:15] ENGINE Bus STARTING 2024-04-26T04:49:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: osdmap e219: 8 total, 8 up, 8 in 2024-04-26T04:49:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: mgrmap e29: y(active, since 1.01649s) 2024-04-26T04:49:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: pgmap v4: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: [26/Apr/2024:04:49:15] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:49:16.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: [26/Apr/2024:04:49:15] ENGINE Serving on http://172.21.15.5:8765 2024-04-26T04:49:16.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[26371]: [26/Apr/2024:04:49:15] ENGINE Bus STARTED 2024-04-26T04:49:16.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: [26/Apr/2024:04:49:15] ENGINE Bus STARTING 2024-04-26T04:49:16.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:16.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:16.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: osdmap e219: 8 total, 8 up, 8 in 2024-04-26T04:49:16.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: mgrmap e29: y(active, since 1.01649s) 2024-04-26T04:49:16.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: pgmap v4: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:16.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: [26/Apr/2024:04:49:15] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:49:16.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: [26/Apr/2024:04:49:15] ENGINE Serving on http://172.21.15.5:8765 2024-04-26T04:49:16.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:16 smithi005 ceph-mon[36961]: [26/Apr/2024:04:49:15] ENGINE Bus STARTED 2024-04-26T04:49:16.984 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: [26/Apr/2024:04:49:15] ENGINE Bus STARTING 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: osdmap e219: 8 total, 8 up, 8 in 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: mgrmap e29: y(active, since 1.01649s) 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: pgmap v4: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: [26/Apr/2024:04:49:15] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:49:16.985 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: [26/Apr/2024:04:49:15] ENGINE Serving on http://172.21.15.5:8765 2024-04-26T04:49:16.986 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-mon[30255]: [26/Apr/2024:04:49:15] ENGINE Bus STARTED 2024-04-26T04:49:17.151 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:49:17.166 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:49:17.166 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:49:17.166 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:49:17.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:16.980+0000 7f24a9cc3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:17.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:17.158+0000 7f24a9cc3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:17.661 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:17.356+0000 7f24a9cc3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:17.661 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:17.660+0000 7f24a9cc3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:17.794 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[26371]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:49:17.794 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[26371]: pgmap v5: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:17.794 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[26371]: osdmap e220: 8 total, 8 up, 8 in 2024-04-26T04:49:17.794 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[26371]: osdmap e221: 8 total, 8 up, 8 in 2024-04-26T04:49:17.794 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1470540851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:17.795 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[36961]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:49:17.795 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[36961]: pgmap v5: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:17.795 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[36961]: osdmap e220: 8 total, 8 up, 8 in 2024-04-26T04:49:17.795 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[36961]: osdmap e221: 8 total, 8 up, 8 in 2024-04-26T04:49:17.795 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:17 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1470540851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:17.926 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-mon[30255]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:49:17.926 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-mon[30255]: pgmap v5: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:17.926 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-mon[30255]: osdmap e220: 8 total, 8 up, 8 in 2024-04-26T04:49:17.926 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-mon[30255]: osdmap e221: 8 total, 8 up, 8 in 2024-04-26T04:49:17.926 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1470540851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:17.927 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:17.761+0000 7f24a9cc3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:18.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:17.966+0000 7f24a9cc3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:18.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:18.164+0000 7f24a9cc3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:18.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:18 smithi005 ceph-mon[26371]: mgrmap e30: y(active, since 3s) 2024-04-26T04:49:18.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:18 smithi005 ceph-mon[26371]: osdmap e222: 8 total, 8 up, 8 in 2024-04-26T04:49:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:18 smithi005 ceph-mon[36961]: mgrmap e30: y(active, since 3s) 2024-04-26T04:49:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:18 smithi005 ceph-mon[36961]: osdmap e222: 8 total, 8 up, 8 in 2024-04-26T04:49:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:18 smithi111 ceph-mon[30255]: mgrmap e30: y(active, since 3s) 2024-04-26T04:49:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:18 smithi111 ceph-mon[30255]: osdmap e222: 8 total, 8 up, 8 in 2024-04-26T04:49:19.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:18.805+0000 7f24a9cc3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:19.159 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:49:19.165 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:49:19.165 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:49:19.174 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:49:19.195 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:49:19.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:19.260+0000 7f24a9cc3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:19.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:19.364+0000 7f24a9cc3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:20.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:19.579+0000 7f24a9cc3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: pgmap v9: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: osdmap e223: 8 total, 8 up, 8 in 2024-04-26T04:49:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/556552725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: pgmap v9: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: osdmap e223: 8 total, 8 up, 8 in 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:20 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/556552725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:20.490 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.131+0000 7f24a9cc3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:20.490 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.232+0000 7f24a9cc3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:20.490 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: pgmap v9: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:20.490 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: osdmap e223: 8 total, 8 up, 8 in 2024-04-26T04:49:20.490 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.491 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.491 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.491 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:20.491 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/556552725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:20.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.489+0000 7f24a9cc3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:20.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.624+0000 7f24a9cc3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:49:20.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.723+0000 7f24a9cc3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:49:21.041 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.850+0000 7f24a9cc3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:21.041 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:20.953+0000 7f24a9cc3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:49:21.041 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:20 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-26T04:49:21.041 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:20 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:21.042 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:20 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:21.042 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 podman[74079]: 2024-04-26 04:49:20.907683974 +0000 UTC m=+0.018377050 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:49:21.042 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 podman[74079]: 2024-04-26 04:49:21.040614712 +0000 UTC m=+0.151307776 container create f53563dec9b4e455cb0ceeb3a089187969320c2cb604f6fc209de5a7c76b1cdb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, version=8.5, vcs-type=git, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, name=ubi8) 2024-04-26T04:49:21.242 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:21.260 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:49:21.267 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:21.376 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-mon[30255]: osdmap e224: 8 total, 8 up, 8 in 2024-04-26T04:49:21.376 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.377 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.377 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.377 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.377 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 podman[74079]: 2024-04-26 04:49:21.284561563 +0000 UTC m=+0.395254625 container init f53563dec9b4e455cb0ceeb3a089187969320c2cb604f6fc209de5a7c76b1cdb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, release=239.1651231664, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., distribution-scope=public, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:49:21.377 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 podman[74079]: 2024-04-26 04:49:21.287292355 +0000 UTC m=+0.397985419 container start f53563dec9b4e455cb0ceeb3a089187969320c2cb604f6fc209de5a7c76b1cdb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, name=ubi8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-type=git, com.redhat.component=ubi8-container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, distribution-scope=public, description=Ceph Grafana Container) 2024-04-26T04:49:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[26371]: osdmap e224: 8 total, 8 up, 8 in 2024-04-26T04:49:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[36961]: osdmap e224: 8 total, 8 up, 8 in 2024-04-26T04:49:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:21 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:21.698 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 bash[74079]: f53563dec9b4e455cb0ceeb3a089187969320c2cb604f6fc209de5a7c76b1cdb 2024-04-26T04:49:21.698 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:21.699 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[74094]: logger=settings t=2024-04-26T04:49:21.443999567Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:49:21.699 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:21 smithi111 podman[74116]: 2024-04-26 04:49:21.474140484 +0000 UTC m=+0.018896945 container died f53563dec9b4e455cb0ceeb3a089187969320c2cb604f6fc209de5a7c76b1cdb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=ubi8-container, distribution-scope=public, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git) 2024-04-26T04:49:22.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:21.871+0000 7f24a9cc3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:49:22.295 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:22 smithi111 podman[74116]: 2024-04-26 04:49:22.030589381 +0000 UTC m=+0.575345840 container remove f53563dec9b4e455cb0ceeb3a089187969320c2cb604f6fc209de5a7c76b1cdb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.openshift.expose-services=, maintainer=Paul Cuzner , vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git) 2024-04-26T04:49:22.295 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:22 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: pgmap v12: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: osdmap e225: 8 total, 8 up, 8 in 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: Standby manager daemon x started 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:22.107+0000 7f24a9cc3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:22] ENGINE Bus STARTING 2024-04-26T04:49:22.296 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: CherryPy Checker: 2024-04-26T04:49:22.297 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: The Application mounted at '' has an empty config. 2024-04-26T04:49:22.297 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: 2024-04-26T04:49:22.297 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:22] ENGINE Serving on http://:::9283 2024-04-26T04:49:22.297 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: [26/Apr/2024:04:49:22] ENGINE Bus STARTED 2024-04-26T04:49:22.667 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:22 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:49:22.717 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: pgmap v12: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:22.717 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: osdmap e225: 8 total, 8 up, 8 in 2024-04-26T04:49:22.717 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: Standby manager daemon x started 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: pgmap v12: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: osdmap e225: 8 total, 8 up, 8 in 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: Standby manager daemon x started 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:22.718 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:49:22.719 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:22 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/4080811447' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[26371]: osdmap e226: 8 total, 8 up, 8 in 2024-04-26T04:49:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[26371]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:49:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:49:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2968859411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:23.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[36961]: osdmap e226: 8 total, 8 up, 8 in 2024-04-26T04:49:23.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[36961]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:49:23.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:49:23.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:23 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2968859411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:23.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:23 smithi111 ceph-mon[30255]: osdmap e226: 8 total, 8 up, 8 in 2024-04-26T04:49:23.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:23 smithi111 ceph-mon[30255]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:49:23.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:23 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:49:23.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:23 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2968859411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:24 smithi005 ceph-mon[26371]: pgmap v15: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 213 op/s 2024-04-26T04:49:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:24 smithi005 ceph-mon[26371]: osdmap e227: 8 total, 8 up, 8 in 2024-04-26T04:49:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:24 smithi005 ceph-mon[36961]: pgmap v15: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 213 op/s 2024-04-26T04:49:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:24 smithi005 ceph-mon[36961]: osdmap e227: 8 total, 8 up, 8 in 2024-04-26T04:49:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:24 smithi111 ceph-mon[30255]: pgmap v15: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 213 op/s 2024-04-26T04:49:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:24 smithi111 ceph-mon[30255]: osdmap e227: 8 total, 8 up, 8 in 2024-04-26T04:49:24.694 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: after_prepare_snap 2024-04-26T04:49:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:25 smithi111 ceph-mon[30255]: osdmap e228: 8 total, 8 up, 8 in 2024-04-26T04:49:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:25 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:25 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:25 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:25 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:25 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/466444913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:25.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[36961]: osdmap e228: 8 total, 8 up, 8 in 2024-04-26T04:49:25.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:25.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/466444913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[26371]: osdmap e228: 8 total, 8 up, 8 in 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:25 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/466444913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:49:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:49:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: pgmap v18: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 213 op/s 2024-04-26T04:49:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: osdmap e229: 8 total, 8 up, 8 in 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:26 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.697 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:49:26.710 INFO:tasks.workunit.client.0.smithi005.stdout:write: * 83886080~1000 2024-04-26T04:49:26.715 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:26.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: pgmap v18: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 213 op/s 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: osdmap e229: 8 total, 8 up, 8 in 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: pgmap v18: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 213 op/s 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: osdmap e229: 8 total, 8 up, 8 in 2024-04-26T04:49:26.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:26 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:26.863 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: pgmap v20: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: osdmap e230: 8 total, 8 up, 8 in 2024-04-26T04:49:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:49:27.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-mon[30255]: osdmap e231: 8 total, 8 up, 8 in 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: pgmap v20: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: osdmap e230: 8 total, 8 up, 8 in 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:49:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[26371]: osdmap e231: 8 total, 8 up, 8 in 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: pgmap v20: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: Reconfiguring daemon prometheus.a on smithi111 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: osdmap e230: 8 total, 8 up, 8 in 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:49:27.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:27 smithi005 ceph-mon[36961]: osdmap e231: 8 total, 8 up, 8 in 2024-04-26T04:49:28.123 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:27 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[63683]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:49:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:49:28.436 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:28 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2027584858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:28.436 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:28 smithi111 ceph-mon[30255]: osdmap e232: 8 total, 8 up, 8 in 2024-04-26T04:49:28.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:28 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2027584858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:28 smithi005 ceph-mon[36961]: osdmap e232: 8 total, 8 up, 8 in 2024-04-26T04:49:28.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:28 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2027584858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:28.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:28 smithi005 ceph-mon[26371]: osdmap e232: 8 total, 8 up, 8 in 2024-04-26T04:49:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-mon[30255]: pgmap v23: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 13 KiB/s wr, 317 op/s 2024-04-26T04:49:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-mon[30255]: osdmap e233: 8 total, 8 up, 8 in 2024-04-26T04:49:29.554 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (13465 ms) 2024-04-26T04:49:29.554 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.Clone 2024-04-26T04:49:29.587 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:29.598 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~8388609 2024-04-26T04:49:29.655 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 16777218~8388609 2024-04-26T04:49:29.705 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap 2024-04-26T04:49:29.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:29 smithi005 ceph-mon[26371]: pgmap v23: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 13 KiB/s wr, 317 op/s 2024-04-26T04:49:29.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:29 smithi005 ceph-mon[26371]: osdmap e233: 8 total, 8 up, 8 in 2024-04-26T04:49:29.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:29 smithi005 ceph-mon[36961]: pgmap v23: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 13 KiB/s wr, 317 op/s 2024-04-26T04:49:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:29 smithi005 ceph-mon[36961]: osdmap e233: 8 total, 8 up, 8 in 2024-04-26T04:49:29.748 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 systemd[1]: Stopping Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:30.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:49:30.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.748Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.749Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.749Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.749Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[71798]: ts=2024-04-26T04:49:29.749Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:49:30.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:29 smithi111 podman[75252]: 2024-04-26 04:49:29.758732938 +0000 UTC m=+0.139257620 container died cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:49:30.399 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 podman[75252]: 2024-04-26 04:49:30.198237876 +0000 UTC m=+0.578762547 container cleanup cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:49:30.400 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 bash[75252]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:49:30.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 podman[75266]: 2024-04-26 04:49:30.399530831 +0000 UTC m=+0.639123392 container remove cf0b4c48ce6a18f54b17191ff8c15ad914aa0cacec44fb924caceb11db906350 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:49:30.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:49:30.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 systemd[1]: Stopped Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:30.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.132s CPU time. 2024-04-26T04:49:30.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 systemd[1]: Starting Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:31.096 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:30 smithi111 ceph-mon[30255]: pgmap v26: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-04-26T04:49:31.096 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:30 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:31.096 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:30 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:31.096 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:30 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3993312145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:31.096 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:30 smithi111 ceph-mon[30255]: osdmap e234: 8 total, 8 up, 8 in 2024-04-26T04:49:31.096 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 podman[75358]: 2024-04-26 04:49:30.73327813 +0000 UTC m=+0.019324486 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:49:31.096 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:30 smithi111 podman[75358]: 2024-04-26 04:49:30.850652321 +0000 UTC m=+0.136698668 container create 9e5fc4ef6c62f89a66c0849c803a561da0d735e51bdd87b1be8754fdf519b462 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:49:31.096 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 podman[75358]: 2024-04-26 04:49:31.069382882 +0000 UTC m=+0.355429229 container init 9e5fc4ef6c62f89a66c0849c803a561da0d735e51bdd87b1be8754fdf519b462 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:49:31.096 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 podman[75358]: 2024-04-26 04:49:31.07193211 +0000 UTC m=+0.357978457 container start 9e5fc4ef6c62f89a66c0849c803a561da0d735e51bdd87b1be8754fdf519b462 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[26371]: pgmap v26: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3993312145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[26371]: osdmap e234: 8 total, 8 up, 8 in 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[36961]: pgmap v26: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-04-26T04:49:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3993312145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:31.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:30 smithi005 ceph-mon[36961]: osdmap e234: 8 total, 8 up, 8 in 2024-04-26T04:49:31.273 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap 2024-04-26T04:49:31.284 INFO:tasks.workunit.client.0.smithi005.stdout:clone image105 -> image108 2024-04-26T04:49:31.376 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:49:31.426 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:49:31.427 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap 2024-04-26T04:49:31.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.095Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.095Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.095Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi111 (none))" 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.095Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.095Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.097Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.097Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.098Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.098Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.100Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.100Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.065µs 2024-04-26T04:49:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.100Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.101Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.119Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.121Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.121Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.121Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.372µs wal_replay_duration=21.04781ms wbl_replay_duration=125ns total_replay_duration=21.086227ms 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.124Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.124Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.124Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.142Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.873914ms db_storage=839ns remote_storage=1.695µs web_handler=438ns query_engine=699ns scrape=417.606µs scrape_sd=84.71µs notify=11.711µs notify_sd=10.84µs rules=17.069595ms tracing=5.6µs 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.142Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:31.142Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 bash[75358]: 9e5fc4ef6c62f89a66c0849c803a561da0d735e51bdd87b1be8754fdf519b462 2024-04-26T04:49:31.520 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:31 smithi111 systemd[1]: Started Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:31] ENGINE Bus STOPPING 2024-04-26T04:49:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:49:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:31] ENGINE Bus STOPPED 2024-04-26T04:49:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:31] ENGINE Bus STARTING 2024-04-26T04:49:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:31] ENGINE Serving on http://:::9283 2024-04-26T04:49:31.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:31] ENGINE Bus STARTED 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: osdmap e235: 8 total, 8 up, 8 in 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:32.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: osdmap e235: 8 total, 8 up, 8 in 2024-04-26T04:49:32.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:32 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:32.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:32 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-26T04:49:32.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:32 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:32.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:32 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi111.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:32.494 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: osdmap e235: 8 total, 8 up, 8 in 2024-04-26T04:49:32.495 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:32 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:33.120 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 podman[75575]: 2024-04-26 04:49:32.967650038 +0000 UTC m=+0.017722893 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:49:33.120 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 podman[75575]: 2024-04-26 04:49:33.12027791 +0000 UTC m=+0.170350763 container create c0482b8228b16dfb75f3c3904c668b8e5a37dfa65618a20786fdd7d280204236 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, release=239.1651231664, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, version=8.5, io.buildah.version=1.32.0) 2024-04-26T04:49:33.317 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap 2024-04-26T04:49:33.329 INFO:tasks.workunit.client.0.smithi005.stdout:clone image108 -> image110 2024-04-26T04:49:33.373 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-mon[30255]: pgmap v29: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 436 op/s 2024-04-26T04:49:33.373 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-mon[30255]: osdmap e236: 8 total, 8 up, 8 in 2024-04-26T04:49:33.373 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2670686441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:33.373 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:33.373 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:33.373 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:33.454 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 1000~1000 2024-04-26T04:49:33.502 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 2010~1000 2024-04-26T04:49:33.504 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:33.637 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 podman[75575]: 2024-04-26 04:49:33.372684965 +0000 UTC m=+0.422757819 container init c0482b8228b16dfb75f3c3904c668b8e5a37dfa65618a20786fdd7d280204236 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, description=Ceph Grafana Container, maintainer=Paul Cuzner , vcs-type=git, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=ubi8, release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, com.redhat.component=ubi8-container, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0) 2024-04-26T04:49:33.638 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 podman[75575]: 2024-04-26 04:49:33.376076281 +0000 UTC m=+0.426149143 container start c0482b8228b16dfb75f3c3904c668b8e5a37dfa65618a20786fdd7d280204236 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vendor=Red Hat, Inc., description=Ceph Grafana Container, release=239.1651231664, maintainer=Paul Cuzner , vcs-type=git, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, version=8.5, io.openshift.tags=base rhel8) 2024-04-26T04:49:33.638 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 bash[75575]: c0482b8228b16dfb75f3c3904c668b8e5a37dfa65618a20786fdd7d280204236 2024-04-26T04:49:33.638 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:33.638 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:33 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[75656]: logger=settings t=2024-04-26T04:49:33.537951392Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:49:33.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[26371]: pgmap v29: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 436 op/s 2024-04-26T04:49:33.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[26371]: osdmap e236: 8 total, 8 up, 8 in 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2670686441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[36961]: pgmap v29: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 436 op/s 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[36961]: osdmap e236: 8 total, 8 up, 8 in 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2670686441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:33 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:33.788 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:33.946 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:34.115 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.Clone (4561 ms) 2024-04-26T04:49:34.115 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneParent 2024-04-26T04:49:34.144 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:34.154 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap 2024-04-26T04:49:34.633 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:34 smithi111 ceph-mon[30255]: Upgrade: Updating mgr.x 2024-04-26T04:49:34.633 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:34 smithi111 ceph-mon[30255]: Deploying daemon mgr.x on smithi111 2024-04-26T04:49:34.633 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:34 smithi111 ceph-mon[30255]: osdmap e237: 8 total, 8 up, 8 in 2024-04-26T04:49:34.633 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:34 smithi111 podman[75703]: 2024-04-26 04:49:34.395117469 +0000 UTC m=+0.849511194 container died c0482b8228b16dfb75f3c3904c668b8e5a37dfa65618a20786fdd7d280204236 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, architecture=x86_64, name=ubi8, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container) 2024-04-26T04:49:34.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:34 smithi005 ceph-mon[26371]: Upgrade: Updating mgr.x 2024-04-26T04:49:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:34 smithi005 ceph-mon[26371]: Deploying daemon mgr.x on smithi111 2024-04-26T04:49:34.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:34 smithi005 ceph-mon[26371]: osdmap e237: 8 total, 8 up, 8 in 2024-04-26T04:49:34.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:34 smithi005 ceph-mon[36961]: Upgrade: Updating mgr.x 2024-04-26T04:49:34.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:34 smithi005 ceph-mon[36961]: Deploying daemon mgr.x on smithi111 2024-04-26T04:49:34.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:34 smithi005 ceph-mon[36961]: osdmap e237: 8 total, 8 up, 8 in 2024-04-26T04:49:35.305 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap 2024-04-26T04:49:35.316 INFO:tasks.workunit.client.0.smithi005.stdout:clone image111 -> image114 2024-04-26T04:49:35.419 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:35.419 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:35 smithi111 ceph-mon[30255]: pgmap v32: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 436 op/s 2024-04-26T04:49:35.420 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:35 smithi111 ceph-mon[30255]: osdmap e238: 8 total, 8 up, 8 in 2024-04-26T04:49:35.420 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:35 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2259527157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:35.420 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:35 smithi111 podman[75703]: 2024-04-26 04:49:35.124176614 +0000 UTC m=+1.578570330 container remove c0482b8228b16dfb75f3c3904c668b8e5a37dfa65618a20786fdd7d280204236 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.openshift.expose-services=, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.component=ubi8-container, name=ubi8, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:49:35.420 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:35 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:49:35.575 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:49:35.700 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneParent (1585 ms) 2024-04-26T04:49:35.700 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-04-26T04:49:35.728 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:35.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:35 smithi005 ceph-mon[36961]: pgmap v32: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 436 op/s 2024-04-26T04:49:35.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:35 smithi005 ceph-mon[36961]: osdmap e238: 8 total, 8 up, 8 in 2024-04-26T04:49:35.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:35 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2259527157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:35.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:35 smithi005 ceph-mon[26371]: pgmap v32: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 436 op/s 2024-04-26T04:49:35.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:35 smithi005 ceph-mon[26371]: osdmap e238: 8 total, 8 up, 8 in 2024-04-26T04:49:35.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:35 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/2259527157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:35.741 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~10 2024-04-26T04:49:35.749 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap 2024-04-26T04:49:36.439 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:36 smithi111 ceph-mon[30255]: osdmap e239: 8 total, 8 up, 8 in 2024-04-26T04:49:36.439 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:36 smithi111 systemd[1]: Stopping Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:36.440 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:36 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:49:36.692 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:36 smithi111 podman[75829]: 2024-04-26 04:49:36.438595857 +0000 UTC m=+0.138008624 container died 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T04:49:36.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:36 smithi005 ceph-mon[26371]: osdmap e239: 8 total, 8 up, 8 in 2024-04-26T04:49:36.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:36 smithi005 ceph-mon[36961]: osdmap e239: 8 total, 8 up, 8 in 2024-04-26T04:49:36.958 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:36 smithi111 podman[75829]: 2024-04-26 04:49:36.765725831 +0000 UTC m=+0.465138579 container cleanup 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T04:49:36.959 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:36 smithi111 bash[75829]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:49:37.154 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap 2024-04-26T04:49:37.166 INFO:tasks.workunit.client.0.smithi005.stdout:clone image115 -> image118 2024-04-26T04:49:37.251 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:37.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:36 smithi111 podman[75843]: 2024-04-26 04:49:36.958605256 +0000 UTC m=+0.518193072 container remove 8666602e6cfca5eeb0f3ef96a9478f4ff40ea51a5b6f0d1b39799e271eaff7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T04:49:37.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:36 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:49:37.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:49:37.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 systemd[1]: Stopped Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:37.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service: Consumed 26.816s CPU time. 2024-04-26T04:49:37.397 INFO:tasks.workunit.client.0.smithi005.stdout:write: Y 0~1 2024-04-26T04:49:37.428 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:37.498 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:37.602 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (1902 ms) 2024-04-26T04:49:37.602 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-04-26T04:49:37.606 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:37 smithi111 ceph-mon[30255]: pgmap v35: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:37.606 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:37 smithi111 ceph-mon[30255]: osdmap e240: 8 total, 8 up, 8 in 2024-04-26T04:49:37.606 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:37 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1487892128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:37.606 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:37 smithi111 ceph-mon[30255]: osdmap e241: 8 total, 8 up, 8 in 2024-04-26T04:49:37.607 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 systemd[1]: Starting Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:37.630 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:37.639 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 41943040~10 2024-04-26T04:49:37.648 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:49:37.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[36961]: pgmap v35: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[36961]: osdmap e240: 8 total, 8 up, 8 in 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1487892128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[36961]: osdmap e241: 8 total, 8 up, 8 in 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[26371]: pgmap v35: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[26371]: osdmap e240: 8 total, 8 up, 8 in 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1487892128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:37.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:37 smithi005 ceph-mon[26371]: osdmap e241: 8 total, 8 up, 8 in 2024-04-26T04:49:37.914 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 podman[75970]: 2024-04-26 04:49:37.509386119 +0000 UTC m=+0.019563912 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:37.914 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 podman[75970]: 2024-04-26 04:49:37.643686037 +0000 UTC m=+0.153863820 container create a0572da61dd7926675f750205df704329e896cf882fd56bd7630e24c54c9ff3e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-26T04:49:37.914 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 podman[75970]: 2024-04-26 04:49:37.914146159 +0000 UTC m=+0.424323949 container init a0572da61dd7926675f750205df704329e896cf882fd56bd7630e24c54c9ff3e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-26T04:49:38.262 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 podman[75970]: 2024-04-26 04:49:37.917237365 +0000 UTC m=+0.427415153 container start a0572da61dd7926675f750205df704329e896cf882fd56bd7630e24c54c9ff3e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2) 2024-04-26T04:49:38.262 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 ceph-mgr[75988]: -- 172.21.15.111:0/592733877 <== mon.1 v2:172.21.15.111:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ecae06a5a0 con 0x55ecae03ec00 2024-04-26T04:49:38.262 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 bash[75970]: a0572da61dd7926675f750205df704329e896cf882fd56bd7630e24c54c9ff3e 2024-04-26T04:49:38.262 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:37 smithi111 systemd[1]: Started Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:38.262 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:38.070+0000 7f8633731200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:38.263 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:38.260+0000 7f8633731200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:38.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:38.372+0000 7f8633731200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:39.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:38.718+0000 7f8633731200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:39.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:38.806+0000 7f8633731200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:39.311 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 41943041~10 2024-04-26T04:49:39.316 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:39.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[36961]: pgmap v38: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 1.16k op/s 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[36961]: osdmap e242: 8 total, 8 up, 8 in 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[26371]: pgmap v38: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 1.16k op/s 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:39 smithi005 ceph-mon[26371]: osdmap e242: 8 total, 8 up, 8 in 2024-04-26T04:49:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:39 smithi111 ceph-mon[30255]: pgmap v38: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 1.16k op/s 2024-04-26T04:49:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:39 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:39 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:39 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:39 smithi111 ceph-mon[30255]: osdmap e242: 8 total, 8 up, 8 in 2024-04-26T04:49:40.230 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:40.054+0000 7f8633731200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:40.230 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:40.228+0000 7f8633731200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:40.460 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 41943042~10 2024-04-26T04:49:40.473 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:49:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-mon[30255]: osdmap e243: 8 total, 8 up, 8 in 2024-04-26T04:49:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3844427614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-mon[30255]: pgmap v41: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 1.16k op/s 2024-04-26T04:49:40.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:40.417+0000 7f8633731200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:40.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:40 smithi005 ceph-mon[26371]: osdmap e243: 8 total, 8 up, 8 in 2024-04-26T04:49:40.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:40 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3844427614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:40.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:40 smithi005 ceph-mon[26371]: pgmap v41: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 1.16k op/s 2024-04-26T04:49:40.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:40 smithi005 ceph-mon[36961]: osdmap e243: 8 total, 8 up, 8 in 2024-04-26T04:49:40.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:40 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3844427614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:40.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:40 smithi005 ceph-mon[36961]: pgmap v41: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 1.16k op/s 2024-04-26T04:49:41.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:40.728+0000 7f8633731200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:41.018 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:40 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:40.833+0000 7f8633731200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:41.316 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:41.040+0000 7f8633731200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:41.317 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:41.254+0000 7f8633731200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:41.634 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:41 smithi111 ceph-mon[30255]: osdmap e244: 8 total, 8 up, 8 in 2024-04-26T04:49:41.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:41 smithi005 ceph-mon[26371]: osdmap e244: 8 total, 8 up, 8 in 2024-04-26T04:49:41.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:41 smithi005 ceph-mon[36961]: osdmap e244: 8 total, 8 up, 8 in 2024-04-26T04:49:42.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:41 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:41.898+0000 7f8633731200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:42.331 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:49:42.359 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:42.413 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:42.630 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[36961]: osdmap e245: 8 total, 8 up, 8 in 2024-04-26T04:49:42.630 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[36961]: pgmap v44: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-04-26T04:49:42.630 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/588697622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[26371]: osdmap e245: 8 total, 8 up, 8 in 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[26371]: pgmap v44: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/588697622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:42.631 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:42 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:42.685 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-mon[30255]: osdmap e245: 8 total, 8 up, 8 in 2024-04-26T04:49:42.685 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-mon[30255]: pgmap v44: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-04-26T04:49:42.685 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/588697622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:42.685 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:42.686 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:42.686 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:42.359+0000 7f8633731200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:42.686 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:42.467+0000 7f8633731200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:42.686 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:42 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:42.683+0000 7f8633731200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:43.444 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (5842 ms) 2024-04-26T04:49:43.444 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-04-26T04:49:43.473 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:49:43] "GET /metrics HTTP/1.1" 200 38847 "" "Prometheus/2.43.0" 2024-04-26T04:49:43.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-mon[26371]: osdmap e246: 8 total, 8 up, 8 in 2024-04-26T04:49:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-mon[36961]: osdmap e246: 8 total, 8 up, 8 in 2024-04-26T04:49:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:43 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:43.484 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 41943040~10 2024-04-26T04:49:43.491 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:49:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-mon[30255]: osdmap e246: 8 total, 8 up, 8 in 2024-04-26T04:49:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:43.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:43.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:43.232+0000 7f8633731200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:43.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:43.331+0000 7f8633731200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:43.857 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:43.603+0000 7f8633731200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:43.857 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:43.747+0000 7f8633731200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:49:43.857 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:43.855+0000 7f8633731200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:49:44.158 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:43 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:43.978+0000 7f8633731200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:44.158 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:44 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:44.076+0000 7f8633731200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:49:44.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:44 smithi111 ceph-mon[30255]: osdmap e247: 8 total, 8 up, 8 in 2024-04-26T04:49:44.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:44 smithi111 ceph-mon[30255]: pgmap v47: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-04-26T04:49:44.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:44 smithi005 ceph-mon[26371]: osdmap e247: 8 total, 8 up, 8 in 2024-04-26T04:49:44.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:44 smithi005 ceph-mon[26371]: pgmap v47: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-04-26T04:49:44.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:44 smithi005 ceph-mon[36961]: osdmap e247: 8 total, 8 up, 8 in 2024-04-26T04:49:44.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:44 smithi005 ceph-mon[36961]: pgmap v47: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-04-26T04:49:45.246 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:44 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:44.907+0000 7f8633731200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:49:45.246 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:45.137+0000 7f8633731200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:49:45.246 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: [26/Apr/2024:04:49:45] ENGINE Bus STARTING 2024-04-26T04:49:45.246 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: CherryPy Checker: 2024-04-26T04:49:45.354 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:49:45.373 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:45.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: osdmap e248: 8 total, 8 up, 8 in 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/4236766356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: Standby manager daemon x restarted 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: Standby manager daemon x started 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: The Application mounted at '' has an empty config. 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:45.519 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: [26/Apr/2024:04:49:45] ENGINE Serving on http://:::9283 2024-04-26T04:49:45.520 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:45 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: [26/Apr/2024:04:49:45] ENGINE Bus STARTED 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: osdmap e248: 8 total, 8 up, 8 in 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/4236766356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: Standby manager daemon x restarted 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: Standby manager daemon x started 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:49:45.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[26371]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: osdmap e248: 8 total, 8 up, 8 in 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/4236766356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: Standby manager daemon x restarted 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: Standby manager daemon x started 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:49:45.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:45 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/753532425' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:49:46.343 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:46 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-26T04:49:46.343 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:46 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:46.344 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:46 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:46.477 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:46.487 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:49:46.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:46 smithi005 ceph-mon[26371]: mgrmap e32: y(active, since 30s), standbys: x 2024-04-26T04:49:46.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:46 smithi005 ceph-mon[26371]: osdmap e249: 8 total, 8 up, 8 in 2024-04-26T04:49:46.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:46 smithi005 ceph-mon[26371]: pgmap v50: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:46.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:46 smithi005 ceph-mon[36961]: mgrmap e32: y(active, since 30s), standbys: x 2024-04-26T04:49:46.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:46 smithi005 ceph-mon[36961]: osdmap e249: 8 total, 8 up, 8 in 2024-04-26T04:49:46.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:46 smithi005 ceph-mon[36961]: pgmap v50: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:46.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:46 smithi111 ceph-mon[30255]: mgrmap e32: y(active, since 30s), standbys: x 2024-04-26T04:49:46.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:46 smithi111 ceph-mon[30255]: osdmap e249: 8 total, 8 up, 8 in 2024-04-26T04:49:46.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:46 smithi111 ceph-mon[30255]: pgmap v50: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:49:47.278 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 podman[79694]: 2024-04-26 04:49:46.927735159 +0000 UTC m=+0.017685627 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:49:47.278 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 podman[79694]: 2024-04-26 04:49:47.047446566 +0000 UTC m=+0.137397034 container create 9bc58253682ccd9eef1fd1d4975277cddca7979fe2ec6925d6217eab34c330ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=ubi8, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, maintainer=Paul Cuzner , io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:49:47.278 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 podman[79694]: 2024-04-26 04:49:47.275125313 +0000 UTC m=+0.365075782 container init 9bc58253682ccd9eef1fd1d4975277cddca7979fe2ec6925d6217eab34c330ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, com.redhat.component=ubi8-container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, name=ubi8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, vendor=Red Hat, Inc.) 2024-04-26T04:49:47.278 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 podman[79694]: 2024-04-26 04:49:47.277760365 +0000 UTC m=+0.367710837 container start 9bc58253682ccd9eef1fd1d4975277cddca7979fe2ec6925d6217eab34c330ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, vcs-type=git, version=8.5, release=239.1651231664, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:49:47.599 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: osdmap e250: 8 total, 8 up, 8 in 2024-04-26T04:49:47.599 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/1158304225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: pgmap v52: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 242 KiB/s rd, 26 KiB/s wr, 334 op/s 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: pgmap v53: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 30 KiB/s wr, 381 op/s 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: Upgrade: Setting container_image for all mgr 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:49:47.600 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[26371]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: osdmap e250: 8 total, 8 up, 8 in 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1158304225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:47.601 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: pgmap v52: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 242 KiB/s rd, 26 KiB/s wr, 334 op/s 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: pgmap v53: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 30 KiB/s wr, 381 op/s 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: Upgrade: Setting container_image for all mgr 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:49:47.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:47 smithi005 ceph-mon[36961]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:49:47.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 bash[79694]: 9bc58253682ccd9eef1fd1d4975277cddca7979fe2ec6925d6217eab34c330ec 2024-04-26T04:49:47.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:47.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[79740]: logger=settings t=2024-04-26T04:49:47.433687463Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:49:47.768 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:47 smithi111 podman[79757]: 2024-04-26 04:49:47.459711561 +0000 UTC m=+0.019014980 container died 9bc58253682ccd9eef1fd1d4975277cddca7979fe2ec6925d6217eab34c330ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, version=8.5, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., release=239.1651231664) 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: osdmap e250: 8 total, 8 up, 8 in 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1158304225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: pgmap v52: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 242 KiB/s rd, 26 KiB/s wr, 334 op/s 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: pgmap v53: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 30 KiB/s wr, 381 op/s 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: Upgrade: Setting container_image for all mgr 2024-04-26T04:49:47.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:47 smithi111 ceph-mon[30255]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:49:48.354 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:48 smithi111 podman[79757]: 2024-04-26 04:49:48.079684434 +0000 UTC m=+0.638987895 container remove 9bc58253682ccd9eef1fd1d4975277cddca7979fe2ec6925d6217eab34c330ec (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Paul Cuzner , description=Ceph Grafana Container, io.buildah.version=1.32.0, name=ubi8, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vcs-type=git, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:49:48.354 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:48 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:49:48.354 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:48 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:49:48.365 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 41943040~10 2024-04-26T04:49:48.371 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap4 2024-04-26T04:49:48.609 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:48 smithi005 ceph-mon[36961]: osdmap e251: 8 total, 8 up, 8 in 2024-04-26T04:49:48.609 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:48 smithi005 ceph-mon[26371]: osdmap e251: 8 total, 8 up, 8 in 2024-04-26T04:49:48.659 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:48 smithi111 ceph-mon[30255]: osdmap e251: 8 total, 8 up, 8 in 2024-04-26T04:49:49.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:48 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: osdmap e252: 8 total, 8 up, 8 in 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: Upgrade: Updating mon.a 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:49.511 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: Deploying daemon mon.a on smithi005 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: pgmap v56: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 30 KiB/s wr, 381 op/s 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3135897422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: osdmap e252: 8 total, 8 up, 8 in 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: Upgrade: Updating mon.a 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: Deploying daemon mon.a on smithi005 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: pgmap v56: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 30 KiB/s wr, 381 op/s 2024-04-26T04:49:49.512 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:49 smithi005 ceph-mon[26371]: from='client.? 172.21.15.5:0/3135897422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: osdmap e252: 8 total, 8 up, 8 in 2024-04-26T04:49:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: Upgrade: Updating mon.a 2024-04-26T04:49:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' 2024-04-26T04:49:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:49.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: Deploying daemon mon.a on smithi005 2024-04-26T04:49:49.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: pgmap v56: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 30 KiB/s wr, 381 op/s 2024-04-26T04:49:49.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:49 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3135897422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:50.386 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 41943040 2024-04-26T04:49:50.411 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:49:50.470 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:49:50.540 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-mon[26371]: osdmap e253: 8 total, 8 up, 8 in 2024-04-26T04:49:50.540 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-mon[36961]: osdmap e253: 8 total, 8 up, 8 in 2024-04-26T04:49:50.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:50 smithi111 ceph-mon[30255]: osdmap e253: 8 total, 8 up, 8 in 2024-04-26T04:49:50.818 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:50 smithi005 systemd[1]: Stopping Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[26367]: 2024-04-26T04:49:50.816+0000 7f743b8d2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[26367]: 2024-04-26T04:49:50.816+0000 7f743b8d2700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 podman[91601]: 2024-04-26 04:49:51.000583146 +0000 UTC m=+0.312155515 container died 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:50] ENGINE Bus STOPPING 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:50] ENGINE Bus STOPPED 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:50] ENGINE Bus STARTING 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:50] ENGINE Serving on http://:::9283 2024-04-26T04:49:51.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:49:50] ENGINE Bus STARTED 2024-04-26T04:49:51.644 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 podman[91601]: 2024-04-26 04:49:51.442106553 +0000 UTC m=+0.753678878 container cleanup 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424) 2024-04-26T04:49:51.644 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 bash[91601]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:49:51.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 podman[91626]: 2024-04-26 04:49:51.643858787 +0000 UTC m=+0.634442037 container remove 6b2f877978294f04327c29fac4f1850f7755329ea5314a33bfd3262465aa8a11 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:51.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:49:51.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 systemd[1]: Stopped Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:51 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service: Consumed 19.140s CPU time. 2024-04-26T04:49:52.432 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 systemd[1]: Starting Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:52.432 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 podman[91781]: 2024-04-26 04:49:52.209674726 +0000 UTC m=+0.018588934 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:52.432 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 podman[91781]: 2024-04-26 04:49:52.320396333 +0000 UTC m=+0.129310528 container create 249028d8f146fb810ee0178b558c8e248685131d22be61c5638cda751b1290ad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T04:49:52.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 podman[91781]: 2024-04-26 04:49:52.539361661 +0000 UTC m=+0.348275857 container init 249028d8f146fb810ee0178b558c8e248685131d22be61c5638cda751b1290ad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T04:49:52.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 podman[91781]: 2024-04-26 04:49:52.543047918 +0000 UTC m=+0.351962122 container start 249028d8f146fb810ee0178b558c8e248685131d22be61c5638cda751b1290ad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:49:52.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: pidfile_write: ignore empty --pid-file 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: load: jerasure load: lrc 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Git sha 0 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: DB SUMMARY 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: DB Session ID: D8XZR8P9W51KWYD8J1VF 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:49:52.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1180 Bytes 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 3443284 ; 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.env: 0x565336d89c60 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.info_log: 0x5653376c9aa0 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:49:52.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.statistics: (nil) 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.use_fsync: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:49:52.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.db_log_dir: 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.wal_dir: 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:49:52.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.write_buffer_manager: 0x56533769c1e0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.unordered_write: 0 2024-04-26T04:49:52.735 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:49:52.736 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:49:52.736 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:49:52.736 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:49:52.736 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.row_cache: None 2024-04-26T04:49:52.736 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.wal_filter: None 2024-04-26T04:49:52.736 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:49:52.737 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.wal_compression: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:49:52.738 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:49:52.739 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_open_files: -1 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Compression algorithms supported: 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kZSTD supported: 0 2024-04-26T04:49:52.740 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.merge_operator: 2024-04-26T04:49:52.741 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_filter: None 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5653376c9ba0) 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: index_type: 0 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: data_block_index_type: 0 2024-04-26T04:49:52.742 INFO:journalctl@ceph.mon.a.smithi005.stdout: index_shortening: 1 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: checksum: 4 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: no_block_cache: 0 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache: 0x5653376cd090 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache_options: 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: capacity : 536870912 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: num_shard_bits : 4 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: strict_capacity_limit : 0 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_cache_compressed: (nil) 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: persistent_cache: (nil) 2024-04-26T04:49:52.743 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_size: 4096 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_size_deviation: 10 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_restart_interval: 16 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: index_block_restart_interval: 1 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: metadata_block_size: 4096 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: partition_filters: 0 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: use_delta_encoding: 1 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: filter_policy: bloomfilter 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: whole_key_filtering: 1 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: verify_compression: 0 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: format_version: 5 2024-04-26T04:49:52.744 INFO:journalctl@ceph.mon.a.smithi005.stdout: enable_index_compression: 1 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout: block_align: 0 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout: max_auto_readahead_size: 262144 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout: prepopulate_block_cache: 0 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression: NoCompression 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:49:52.745 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.num_levels: 7 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:49:52.746 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:49:52.747 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:49:52.749 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:49:52.749 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:49:52.749 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:49:52.749 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:49:52.749 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:49:52.750 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:49:52.751 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:49:52.752 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.ttl: 2592000 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:49:52.753 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12773, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ce1e4591-b70c-4ab2-80a4-35003e9d0be7 2024-04-26T04:49:52.754 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106992573948, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106992584117, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 3219694, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12774, "largest_seqno": 14307, "table_properties": {"data_size": 3211223, "index_size": 5724, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 16004, "raw_average_key_size": 23, "raw_value_size": 3195545, "raw_average_value_size": 4734, "num_data_blocks": 253, "num_entries": 675, "num_filter_entries": 675, "num_deletions": 6, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106992, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ce1e4591-b70c-4ab2-80a4-35003e9d0be7", "db_session_id": "D8XZR8P9W51KWYD8J1VF", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106992584236, "job": 1, "event": "recovery_finished"} 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5653377be000 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: DB pointer 0x5653377a8000 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** DB Stats ** 2024-04-26T04:49:52.755 INFO:journalctl@ceph.mon.a.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:49:52.756 INFO:journalctl@ceph.mon.a.smithi005.stdout: L0 1/0 3.07 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 623.0 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: L6 1/0 7.71 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: Sum 2/0 10.78 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 623.0 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 623.0 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 623.0 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:49:52.757 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: Cumulative compaction: 0.00 GB write, 241.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: Interval compaction: 0.00 GB write, 241.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:52.758 INFO:journalctl@ceph.mon.a.smithi005.stdout: Block cache BinnedLRUCache@0x5653376cd090#2 capacity: 512.00 MB usage: 7.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: starting mon.a rank 0 at public addrs [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] at bind addrs [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???) e3 preinit fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).mds e9 new map 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).mds e9 print_map 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: e9 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: legacy client fscid: 1 2024-04-26T04:49:52.759 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: Filesystem 'foo' (1) 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: fs_name foo 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: epoch 8 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: created 2024-04-26T04:45:21.622880+0000 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: modified 2024-04-26T04:45:39.686257+0000 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: tableserver 0 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: root 0 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: session_timeout 60 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: session_autoclose 300 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: max_file_size 1099511627776 2024-04-26T04:49:52.760 INFO:journalctl@ceph.mon.a.smithi005.stdout: max_xattr_size 65536 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: required_client_features {} 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: last_failure 0 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: last_failure_osd_epoch 0 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: max_mds 1 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: in 0 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: up {0=24554} 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: failed 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: damaged 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: stopped 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: data_pools [4] 2024-04-26T04:49:52.761 INFO:journalctl@ceph.mon.a.smithi005.stdout: metadata_pool 3 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: inline_data disabled 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: balancer 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: bal_rank_mask -1 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: standby_count_wanted 1 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: [mds.foo.smithi005.zdnlsb{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: Standby daemons: 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout: [mds.foo.smithi111.furhfd{-1:24538} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:49:52.762 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).osd e254 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:49:52.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).osd e254 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:49:52.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).osd e254 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:49:52.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).osd e254 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:49:52.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 ceph-mon[91801]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-04-26T04:49:52.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 bash[91781]: 249028d8f146fb810ee0178b558c8e248685131d22be61c5638cda751b1290ad 2024-04-26T04:49:52.763 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:52 smithi005 systemd[1]: Started Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:49:53] "GET /metrics HTTP/1.1" 200 38851 "" "Prometheus/2.43.0" 2024-04-26T04:49:54.129 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-26T04:49:54.214 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (10770 ms) 2024-04-26T04:49:54.214 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-04-26T04:49:54.245 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:54.256 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: pgmap v59: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 214 KiB/s rd, 16 KiB/s wr, 275 op/s 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1921385957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: pgmap v60: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 12 KiB/s wr, 200 op/s 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: mon.a calling monitor election 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: monmap epoch 3 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: last_changed 2024-04-26T04:40:01.288521+0000 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: min_mon_release 17 (quincy) 2024-04-26T04:49:54.397 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: election_strategy: 1 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: osdmap e254: 8 total, 8 up, 8 in 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: daemon grafana.a on smithi111 is in unknown state 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: application not enabled on pool 'test-librbd-smithi005-77850-9' 2024-04-26T04:49:54.398 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: from='mgr.15291 ' entity='' 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mon[30255]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: ignoring --setuser ceph since I am not root 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: ignoring --setgroup ceph since I am not root 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-mgr[75988]: -- 172.21.15.111:0/2329248515 <== mon.1 v2:172.21.15.111:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560075c885a0 con 0x560075c5c800 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:54.200+0000 7fe40b790200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:54.399 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:54.395+0000 7fe40b790200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:54.435 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.AbortInUseImage (221 ms) 2024-04-26T04:49:54.435 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-04-26T04:49:54.465 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:54.465 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:54.465 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.465 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:54.465 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: pgmap v59: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 214 KiB/s rd, 16 KiB/s wr, 275 op/s 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/1921385957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: pgmap v60: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 12 KiB/s wr, 200 op/s 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: mon.a calling monitor election 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: monmap epoch 3 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: last_changed 2024-04-26T04:40:01.288521+0000 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: min_mon_release 17 (quincy) 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: election_strategy: 1 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:49:54.466 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: osdmap e254: 8 total, 8 up, 8 in 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: daemon grafana.a on smithi111 is in unknown state 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: application not enabled on pool 'test-librbd-smithi005-77850-9' 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: from='mgr.15291 ' entity='' 2024-04-26T04:49:54.467 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[36961]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: from='mgr.15291 172.21.15.5:0/4063948764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: pgmap v59: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 214 KiB/s rd, 16 KiB/s wr, 275 op/s 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1921385957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: pgmap v60: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 12 KiB/s wr, 200 op/s 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: mon.a calling monitor election 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: monmap epoch 3 2024-04-26T04:49:54.468 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: last_changed 2024-04-26T04:40:01.288521+0000 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: min_mon_release 17 (quincy) 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: election_strategy: 1 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:49:54.469 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: osdmap e254: 8 total, 8 up, 8 in 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: mgrmap e32: y(active, since 39s), standbys: x 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: daemon grafana.a on smithi111 is in unknown state 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: application not enabled on pool 'test-librbd-smithi005-77850-9' 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: from='mgr.15291 ' entity='' 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mon[91801]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T04:49:54.470 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ignoring --setuser ceph since I am not root 2024-04-26T04:49:54.471 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ignoring --setgroup ceph since I am not root 2024-04-26T04:49:54.471 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-mgr[83194]: -- 172.21.15.5:0/2184730435 <== mon.0 v2:172.21.15.5:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561ac57665a0 con 0x561ac573b000 2024-04-26T04:49:54.471 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:54.239+0000 7f1620962200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:54.471 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:54.463+0000 7f1620962200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:54.476 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:49:54.528 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:49:54.581 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:49:54.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:54.592+0000 7f1620962200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:54.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:54.517+0000 7fe40b790200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:54.770 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:49:54.811 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:49:54.878 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:49:54.937 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:49:55.071 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (636 ms) 2024-04-26T04:49:55.071 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-04-26T04:49:55.107 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:49:55.120 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:49:55.183 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:49:55.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:55 smithi005 ceph-mon[91801]: osdmap e255: 8 total, 8 up, 8 in 2024-04-26T04:49:55.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:55 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2837570905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:55 smithi005 ceph-mon[36961]: osdmap e255: 8 total, 8 up, 8 in 2024-04-26T04:49:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:55 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/2837570905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:55.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:54.993+0000 7f1620962200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:55.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:55.118+0000 7f1620962200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:55.247 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:55 smithi111 ceph-mon[30255]: osdmap e255: 8 total, 8 up, 8 in 2024-04-26T04:49:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:55 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2837570905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:55.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:54.880+0000 7fe40b790200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:55.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:54.977+0000 7fe40b790200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:56.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:56.191+0000 7fe40b790200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:56.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:56.360+0000 7fe40b790200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:56.557 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:56.411+0000 7f1620962200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:56.823 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:56.532+0000 7fe40b790200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:56.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:56.593+0000 7f1620962200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:56.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:56 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:56.775+0000 7f1620962200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:57.082 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 8388609~8388609 2024-04-26T04:49:57.099 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 0~1000 2024-04-26T04:49:57.099 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1010~1000 2024-04-26T04:49:57.099 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:49:57.136 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:57 smithi111 ceph-mon[30255]: osdmap e256: 8 total, 8 up, 8 in 2024-04-26T04:49:57.136 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:56.822+0000 7fe40b790200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:57.136 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:56.915+0000 7fe40b790200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:57.137 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:57.134+0000 7fe40b790200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:57.408 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:57 smithi005 ceph-mon[91801]: osdmap e256: 8 total, 8 up, 8 in 2024-04-26T04:49:57.409 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:57 smithi005 ceph-mon[36961]: osdmap e256: 8 total, 8 up, 8 in 2024-04-26T04:49:57.409 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:57 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:57.085+0000 7f1620962200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:57.409 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:57 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:57.195+0000 7f1620962200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:57.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:57.329+0000 7fe40b790200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:57.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:57 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:57.407+0000 7f1620962200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:57.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:57 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:57.614+0000 7f1620962200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:58 smithi111 ceph-mon[30255]: osdmap e257: 8 total, 8 up, 8 in 2024-04-26T04:49:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:58 smithi111 ceph-mon[30255]: osdmap e258: 8 total, 8 up, 8 in 2024-04-26T04:49:58.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:57.935+0000 7fe40b790200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:58.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-mon[91801]: osdmap e257: 8 total, 8 up, 8 in 2024-04-26T04:49:58.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-mon[91801]: osdmap e258: 8 total, 8 up, 8 in 2024-04-26T04:49:58.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-mon[36961]: osdmap e257: 8 total, 8 up, 8 in 2024-04-26T04:49:58.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-mon[36961]: osdmap e258: 8 total, 8 up, 8 in 2024-04-26T04:49:58.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:58.248+0000 7f1620962200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:58.607 INFO:tasks.workunit.client.0.smithi005.stdout:write: X 8388609~1000 2024-04-26T04:49:58.612 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16777228~1000 2024-04-26T04:49:58.613 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 167772160 2024-04-26T04:49:58.620 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 83886079~8388609 2024-04-26T04:49:58.646 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:49:58.660 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:58 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-26T04:49:58.660 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:58 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:58.660 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:58 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:58.660 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:58.365+0000 7fe40b790200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:58.660 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:58.463+0000 7fe40b790200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:58.912 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:49:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:49:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:49:58.912 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:58.710+0000 7fe40b790200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:58.913 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:58 smithi111 podman[79924]: 2024-04-26 04:49:58.671626026 +0000 UTC m=+0.028859699 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:49:58.913 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:58 smithi111 podman[79924]: 2024-04-26 04:49:58.796913008 +0000 UTC m=+0.154146727 container create 02d437159f6a0c0e741308584cde6034a48032ae8ccfc685779e3d1eaaff8364 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, description=Ceph Grafana Container, name=ubi8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, release=239.1651231664, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:49:58.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:58.710+0000 7f1620962200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:58.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:58 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:58.811+0000 7f1620962200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:59.171 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 podman[79924]: 2024-04-26 04:49:59.007517129 +0000 UTC m=+0.364750801 container init 02d437159f6a0c0e741308584cde6034a48032ae8ccfc685779e3d1eaaff8364 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., distribution-scope=public, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.expose-services=, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:49:59.172 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 podman[79924]: 2024-04-26 04:49:59.010194574 +0000 UTC m=+0.367428247 container start 02d437159f6a0c0e741308584cde6034a48032ae8ccfc685779e3d1eaaff8364 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, name=ubi8, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, architecture=x86_64, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:49:59.172 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 bash[79924]: 02d437159f6a0c0e741308584cde6034a48032ae8ccfc685779e3d1eaaff8364 2024-04-26T04:49:59.172 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:59.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:59.023+0000 7f1620962200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:59.487 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:59.283+0000 7fe40b790200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:59.487 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:59.382+0000 7fe40b790200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:59.487 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[79939]: logger=settings t=2024-04-26T04:49:59.171385178Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:49:59.487 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 podman[79959]: 2024-04-26 04:49:59.201281098 +0000 UTC m=+0.021611193 container died 02d437159f6a0c0e741308584cde6034a48032ae8ccfc685779e3d1eaaff8364 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.32.0, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, release=239.1651231664, vendor=Red Hat, Inc., name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:49:59.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-mon[30255]: osdmap e259: 8 total, 8 up, 8 in 2024-04-26T04:49:59.768 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:59.647+0000 7fe40b790200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:59.933 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:49:59 smithi005 ceph-mon[91801]: osdmap e259: 8 total, 8 up, 8 in 2024-04-26T04:49:59.934 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:49:59 smithi005 ceph-mon[36961]: osdmap e259: 8 total, 8 up, 8 in 2024-04-26T04:49:59.934 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:59.572+0000 7f1620962200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:59.934 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:59.675+0000 7f1620962200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:50:00.056 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 podman[79959]: 2024-04-26 04:49:59.774040841 +0000 UTC m=+0.594370935 container remove 02d437159f6a0c0e741308584cde6034a48032ae8ccfc685779e3d1eaaff8364 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, version=8.5, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vcs-type=git, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=) 2024-04-26T04:50:00.057 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:49:59 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:50:00.057 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:59.790+0000 7fe40b790200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:50:00.057 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:49:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:49:59.903+0000 7fe40b790200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:50:00.057 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:50:00.055+0000 7fe40b790200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:50:00.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:49:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:49:59.932+0000 7f1620962200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:50:00.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:00.072+0000 7f1620962200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:50:00.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:00.174+0000 7f1620962200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:50:00.518 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:50:00.157+0000 7fe40b790200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:50:00.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:00 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:50:00.601 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:00.296+0000 7f1620962200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:50:00.601 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:00.393+0000 7f1620962200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:50:00.620 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:00.640 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83886079~1 2024-04-26T04:50:00.648 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:00.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-mon[91801]: osdmap e260: 8 total, 8 up, 8 in 2024-04-26T04:50:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-mon[91801]: overall HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-mon[36961]: osdmap e260: 8 total, 8 up, 8 in 2024-04-26T04:50:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:00 smithi005 ceph-mon[36961]: overall HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.992 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:00 smithi111 ceph-mon[30255]: osdmap e260: 8 total, 8 up, 8 in 2024-04-26T04:50:00.992 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:00 smithi111 ceph-mon[30255]: overall HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:50:01.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:50:00.990+0000 7fe40b790200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:50:01.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:50:01.212+0000 7fe40b790200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:50:01.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: [26/Apr/2024:04:50:01] ENGINE Bus STARTING 2024-04-26T04:50:01.585 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.260+0000 7f1620962200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:50:01.585 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.489+0000 7f1620962200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: CherryPy Checker: 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: The Application mounted at '' has an empty config. 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: [26/Apr/2024:04:50:01] ENGINE Serving on http://:::9283 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x[75984]: [26/Apr/2024:04:50:01] ENGINE Bus STARTED 2024-04-26T04:50:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: osdmap e261: 8 total, 8 up, 8 in 2024-04-26T04:50:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: Standby manager daemon x restarted 2024-04-26T04:50:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: Standby manager daemon x started 2024-04-26T04:50:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:50:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:50:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: Active manager daemon y restarted 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: Activating manager daemon y 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: osdmap e262: 8 total, 8 up, 8 in 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: mgrmap e34: y(active, starting, since 0.00879937s), standbys: x 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:50:01.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:50:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: osdmap e261: 8 total, 8 up, 8 in 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: Standby manager daemon x restarted 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: Standby manager daemon x started 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: Active manager daemon y restarted 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: Activating manager daemon y 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: osdmap e262: 8 total, 8 up, 8 in 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: mgrmap e34: y(active, starting, since 0.00879937s), standbys: x 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:01.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:50:01.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: Manager daemon y is now available 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:01.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: Manager daemon y is now available 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:50:01] ENGINE Bus STARTING 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.637+0000 7f157fc90700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.637+0000 7f157fc90700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.986 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.637+0000 7f157fc90700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.637+0000 7f157fc90700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.637+0000 7f157fc90700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.638+0000 7f1585c9c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.638+0000 7f1585c9c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.638+0000 7f1585c9c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.638+0000 7f1585c9c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.638+0000 7f1585c9c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: CherryPy Checker: 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: The Application mounted at '' has an empty config. 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:50:01] ENGINE Serving on http://:::9283 2024-04-26T04:50:01.987 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:01 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:50:01] ENGINE Bus STARTED 2024-04-26T04:50:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: osdmap e261: 8 total, 8 up, 8 in 2024-04-26T04:50:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: Standby manager daemon x restarted 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: Standby manager daemon x started 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.? 172.21.15.111:0/2967954955' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: Active manager daemon y restarted 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: Activating manager daemon y 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: osdmap e262: 8 total, 8 up, 8 in 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: mgrmap e34: y(active, starting, since 0.00879937s), standbys: x 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi111.furhfd"}]: dispatch 2024-04-26T04:50:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi005.zdnlsb"}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: Manager daemon y is now available 2024-04-26T04:50:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:02.021 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:02.021 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:02.021 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:01 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:alertmanager.a smithi005 *:9093,9094 running (88s) 42s ago 5m 17.6M - 0.25.0 c8568f914cd2 3afeca478730 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:grafana.a smithi111 *:3000 unknown 20s ago 5m - - 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi005.zdnlsb smithi005 running (4m) 42s ago 4m 18.3M - 17.2.7-902-gc61472c8 0de407c1b3af 7ddb93a6b5a1 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi111.furhfd smithi111 running (4m) 20s ago 4m 16.1M - 17.2.7-902-gc61472c8 0de407c1b3af 3e206ca72f2e 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.x smithi111 *:8443,9283,8765 running (25s) 20s ago 9m 65.9M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0572da61dd7 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.y smithi005 *:8443,9283,8765 running (57s) 42s ago 11m 481M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c895550b1a50 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:mon.a smithi005 running (11m) 42s ago 11m 57.0M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 6b2f87797829 2024-04-26T04:50:02.530 INFO:teuthology.orchestra.run.smithi005.stdout:mon.b smithi111 running (10m) 20s ago 10m 48.5M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 20923c1c0e03 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:mon.c smithi005 running (10m) 42s ago 10m 45.2M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 78e8c297f4e2 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.a smithi005 *:9100 running (85s) 42s ago 5m 9629k - 1.5.0 0da6a335fe13 c74ab514255d 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.b smithi111 *:9100 running (63s) 20s ago 5m 8212k - 1.5.0 0da6a335fe13 cc0664bb12c9 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.0 smithi005 running (9m) 42s ago 9m 132M 2481M 17.2.7-902-gc61472c8 0de407c1b3af b99cc094c6d7 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.1 smithi005 running (9m) 42s ago 9m 102M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 9e578b598142 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.2 smithi005 running (8m) 42s ago 8m 104M 2481M 17.2.7-902-gc61472c8 0de407c1b3af f25e1cd874ca 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.3 smithi005 running (8m) 42s ago 8m 99.4M 2481M 17.2.7-902-gc61472c8 0de407c1b3af fd23fb067fc9 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.4 smithi111 running (7m) 20s ago 7m 119M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e2994c2a7aa2 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.5 smithi111 running (7m) 20s ago 7m 99.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5da203330767 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.6 smithi111 running (7m) 20s ago 7m 118M 2480M 17.2.7-902-gc61472c8 0de407c1b3af b7798f2bbff0 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:osd.7 smithi111 running (6m) 20s ago 6m 86.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 60f1bafb36a6 2024-04-26T04:50:02.531 INFO:teuthology.orchestra.run.smithi005.stdout:prometheus.a smithi111 *:9095 running (31s) 20s ago 6m 41.9M - 2.43.0 a07b618ecd1d 9e5fc4ef6c62 2024-04-26T04:50:02.871 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:50:02.516+0000 7f15dde08700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: [26/Apr/2024:04:50:02] ENGINE Bus STARTING 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: osdmap e263: 8 total, 8 up, 8 in 2024-04-26T04:50:02.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[91801]: mgrmap e35: y(active, since 1.01679s), standbys: x 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: [26/Apr/2024:04:50:02] ENGINE Bus STARTING 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: osdmap e263: 8 total, 8 up, 8 in 2024-04-26T04:50:02.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:02 smithi005 ceph-mon[36961]: mgrmap e35: y(active, since 1.01679s), standbys: x 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: [26/Apr/2024:04:50:02] ENGINE Bus STARTING 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: osdmap e263: 8 total, 8 up, 8 in 2024-04-26T04:50:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:02 smithi111 ceph-mon[30255]: mgrmap e35: y(active, since 1.01679s), standbys: x 2024-04-26T04:50:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:50:03] "GET /metrics HTTP/1.1" 200 36027 "" "Prometheus/2.43.0" 2024-04-26T04:50:03.662 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: [26/Apr/2024:04:50:02] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: [26/Apr/2024:04:50:02] ENGINE Serving on http://172.21.15.5:8765 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: [26/Apr/2024:04:50:02] ENGINE Bus STARTED 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: from='client.25367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: pgmap v4: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[91801]: osdmap e264: 8 total, 8 up, 8 in 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: [26/Apr/2024:04:50:02] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:50:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: [26/Apr/2024:04:50:02] ENGINE Serving on http://172.21.15.5:8765 2024-04-26T04:50:03.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: [26/Apr/2024:04:50:02] ENGINE Bus STARTED 2024-04-26T04:50:03.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: from='client.25367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:03.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: pgmap v4: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:03.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:50:03.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:03 smithi005 ceph-mon[36961]: osdmap e264: 8 total, 8 up, 8 in 2024-04-26T04:50:03.989 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:50:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: [26/Apr/2024:04:50:02] ENGINE Serving on https://172.21.15.5:7150 2024-04-26T04:50:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: [26/Apr/2024:04:50:02] ENGINE Serving on http://172.21.15.5:8765 2024-04-26T04:50:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: [26/Apr/2024:04:50:02] ENGINE Bus STARTED 2024-04-26T04:50:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: from='client.25367 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: pgmap v4: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:50:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:03 smithi111 ceph-mon[30255]: osdmap e264: 8 total, 8 up, 8 in 2024-04-26T04:50:04.095 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:50:04.131 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:04.630 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 0~8388609 2024-04-26T04:50:04.641 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 16777218~8388609 2024-04-26T04:50:04.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:04 smithi005 ceph-mon[91801]: pgmap v5: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:04.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:04 smithi005 ceph-mon[91801]: mgrmap e36: y(active, since 2s), standbys: x 2024-04-26T04:50:04.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:04 smithi005 ceph-mon[36961]: pgmap v5: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:04.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:04 smithi005 ceph-mon[36961]: mgrmap e36: y(active, since 2s), standbys: x 2024-04-26T04:50:04.741 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:04.947 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:04 smithi111 ceph-mon[30255]: pgmap v5: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:04.947 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:04 smithi111 ceph-mon[30255]: mgrmap e36: y(active, since 2s), standbys: x 2024-04-26T04:50:05.948 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:05 smithi005 ceph-mon[91801]: osdmap e265: 8 total, 8 up, 8 in 2024-04-26T04:50:05.949 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:05 smithi005 ceph-mon[36961]: osdmap e265: 8 total, 8 up, 8 in 2024-04-26T04:50:06.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:05 smithi111 ceph-mon[30255]: osdmap e265: 8 total, 8 up, 8 in 2024-04-26T04:50:06.603 INFO:teuthology.orchestra.run.smithi005.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[91801]: pgmap v8: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[91801]: mgrmap e37: y(active, since 4s), standbys: x 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[91801]: osdmap e266: 8 total, 8 up, 8 in 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[36961]: pgmap v8: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[36961]: mgrmap e37: y(active, since 4s), standbys: x 2024-04-26T04:50:06.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[36961]: osdmap e266: 8 total, 8 up, 8 in 2024-04-26T04:50:06.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:06.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:06.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:06 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:06 smithi111 ceph-mon[30255]: pgmap v8: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:06 smithi111 ceph-mon[30255]: mgrmap e37: y(active, since 4s), standbys: x 2024-04-26T04:50:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:06 smithi111 ceph-mon[30255]: osdmap e266: 8 total, 8 up, 8 in 2024-04-26T04:50:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:06 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:06 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:06 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.488 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:50:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[36961]: from='client.25406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[36961]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[36961]: osdmap e267: 8 total, 8 up, 8 in 2024-04-26T04:50:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[36961]: osdmap e268: 8 total, 8 up, 8 in 2024-04-26T04:50:07.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[91801]: from='client.25406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:07.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[91801]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:07.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[91801]: osdmap e267: 8 total, 8 up, 8 in 2024-04-26T04:50:07.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:07.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:07 smithi005 ceph-mon[91801]: osdmap e268: 8 total, 8 up, 8 in 2024-04-26T04:50:07.861 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:07 smithi111 ceph-mon[30255]: from='client.25406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:07 smithi111 ceph-mon[30255]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:07 smithi111 ceph-mon[30255]: osdmap e267: 8 total, 8 up, 8 in 2024-04-26T04:50:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:07 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:07 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:07 smithi111 ceph-mon[30255]: osdmap e268: 8 total, 8 up, 8 in 2024-04-26T04:50:08.944 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.AbortWithSnapshots (13873 ms) 2024-04-26T04:50:08.944 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-04-26T04:50:08.973 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:08.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:08 smithi005 ceph-mon[91801]: pgmap v11: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:08.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:08 smithi005 ceph-mon[91801]: osdmap e269: 8 total, 8 up, 8 in 2024-04-26T04:50:08.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:08 smithi005 ceph-mon[36961]: pgmap v11: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:08.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:08 smithi005 ceph-mon[36961]: osdmap e269: 8 total, 8 up, 8 in 2024-04-26T04:50:08.984 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:08.993 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:08 smithi111 ceph-mon[30255]: pgmap v11: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:50:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:08 smithi111 ceph-mon[30255]: osdmap e269: 8 total, 8 up, 8 in 2024-04-26T04:50:09.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:50:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:50:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:50:10.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-26T04:50:10.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:10.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:10.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 podman[82964]: 2024-04-26 04:50:10.373746511 +0000 UTC m=+0.017992708 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:50:10.494 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 podman[82964]: 2024-04-26 04:50:10.493362549 +0000 UTC m=+0.137608729 container create b803d1ddc363f8e28e72a31b8e2dda47443262fe0a3695d4ab2ff36ad7c4df66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.buildah.version=1.32.0, release=239.1651231664, io.openshift.expose-services=, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:50:10.659 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:10.707 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:10 smithi111 ceph-mon[30255]: pgmap v14: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 414 op/s 2024-04-26T04:50:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:10 smithi111 ceph-mon[30255]: osdmap e270: 8 total, 8 up, 8 in 2024-04-26T04:50:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:10 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:10 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:10.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[91801]: pgmap v14: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 414 op/s 2024-04-26T04:50:10.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[91801]: osdmap e270: 8 total, 8 up, 8 in 2024-04-26T04:50:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[36961]: pgmap v14: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 414 op/s 2024-04-26T04:50:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[36961]: osdmap e270: 8 total, 8 up, 8 in 2024-04-26T04:50:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:10 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.225 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 podman[82964]: 2024-04-26 04:50:10.800802178 +0000 UTC m=+0.445048359 container init b803d1ddc363f8e28e72a31b8e2dda47443262fe0a3695d4ab2ff36ad7c4df66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, distribution-scope=public, name=ubi8, architecture=x86_64, com.redhat.component=ubi8-container, version=8.5, io.openshift.tags=base rhel8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664) 2024-04-26T04:50:11.225 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 podman[82964]: 2024-04-26 04:50:10.804041527 +0000 UTC m=+0.448287713 container start b803d1ddc363f8e28e72a31b8e2dda47443262fe0a3695d4ab2ff36ad7c4df66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, version=8.5, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:50:11.225 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 bash[82964]: b803d1ddc363f8e28e72a31b8e2dda47443262fe0a3695d4ab2ff36ad7c4df66 2024-04-26T04:50:11.226 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:11.226 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[83313]: logger=settings t=2024-04-26T04:50:10.9540229Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:50:11.860 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: osdmap e271: 8 total, 8 up, 8 in 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:11 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:11.928 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:11 smithi111 podman[83330]: 2024-04-26 04:50:11.61142361 +0000 UTC m=+0.649632660 container died b803d1ddc363f8e28e72a31b8e2dda47443262fe0a3695d4ab2ff36ad7c4df66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, architecture=x86_64, description=Ceph Grafana Container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:50:11.937 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:50:11.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: osdmap e271: 8 total, 8 up, 8 in 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: osdmap e271: 8 total, 8 up, 8 in 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:11 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:12.653 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:12 smithi111 podman[83330]: 2024-04-26 04:50:12.28158328 +0000 UTC m=+1.319792324 container remove b803d1ddc363f8e28e72a31b8e2dda47443262fe0a3695d4ab2ff36ad7c4df66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, name=ubi8, io.openshift.expose-services=, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , version=8.5, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64) 2024-04-26T04:50:12.653 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:12 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:50:12.653 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:12 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: pgmap v17: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 414 op/s 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: osdmap e272: 8 total, 8 up, 8 in 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: pgmap v17: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 414 op/s 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: osdmap e272: 8 total, 8 up, 8 in 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:12.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: pgmap v17: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 414 op/s 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.conf 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.conf 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: osdmap e272: 8 total, 8 up, 8 in 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:12 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.084 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV1Parent (4140 ms) 2024-04-26T04:50:13.084 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-04-26T04:50:13.114 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:13.125 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:13.133 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:13.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:50:13] "GET /metrics HTTP/1.1" 200 36027 "" "Prometheus/2.43.0" 2024-04-26T04:50:13.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: pgmap v19: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 21 KiB/s wr, 449 op/s 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: pgmap v20: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 44 MiB/s rd, 26 KiB/s wr, 557 op/s 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s rd, 27 KiB/s wr, 580 op/s 2024-04-26T04:50:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[91801]: osdmap e273: 8 total, 8 up, 8 in 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: pgmap v19: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 21 KiB/s wr, 449 op/s 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:13.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:13.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: pgmap v20: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 44 MiB/s rd, 26 KiB/s wr, 557 op/s 2024-04-26T04:50:13.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s rd, 27 KiB/s wr, 580 op/s 2024-04-26T04:50:13.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:13.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:13 smithi005 ceph-mon[36961]: osdmap e273: 8 total, 8 up, 8 in 2024-04-26T04:50:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: Updating smithi111:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: Updating smithi005:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: Updating smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: Updating smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:50:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: pgmap v19: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 21 KiB/s wr, 449 op/s 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: pgmap v20: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 44 MiB/s rd, 26 KiB/s wr, 557 op/s 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s rd, 27 KiB/s wr, 580 op/s 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:50:14.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:13 smithi111 ceph-mon[30255]: osdmap e273: 8 total, 8 up, 8 in 2024-04-26T04:50:14.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:14 smithi005 ceph-mon[36961]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:50:14.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:14 smithi005 ceph-mon[36961]: osdmap e274: 8 total, 8 up, 8 in 2024-04-26T04:50:14.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:14 smithi005 ceph-mon[91801]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:50:14.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:14 smithi005 ceph-mon[91801]: osdmap e274: 8 total, 8 up, 8 in 2024-04-26T04:50:14.990 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:14 smithi111 ceph-mon[30255]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:50:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:14 smithi111 ceph-mon[30255]: osdmap e274: 8 total, 8 up, 8 in 2024-04-26T04:50:15.055 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:16.211 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:50:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:15 smithi005 ceph-mon[91801]: pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s rd, 27 KiB/s wr, 582 op/s 2024-04-26T04:50:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:15 smithi005 ceph-mon[91801]: osdmap e275: 8 total, 8 up, 8 in 2024-04-26T04:50:16.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:15 smithi005 ceph-mon[36961]: pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s rd, 27 KiB/s wr, 582 op/s 2024-04-26T04:50:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:15 smithi005 ceph-mon[36961]: osdmap e275: 8 total, 8 up, 8 in 2024-04-26T04:50:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:15 smithi111 ceph-mon[30255]: pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s rd, 27 KiB/s wr, 582 op/s 2024-04-26T04:50:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:15 smithi111 ceph-mon[30255]: osdmap e275: 8 total, 8 up, 8 in 2024-04-26T04:50:16.292 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:50:17.114 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV2Parent (4030 ms) 2024-04-26T04:50:17.114 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-04-26T04:50:17.146 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:17.157 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:17.166 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:16 smithi111 ceph-mon[30255]: osdmap e276: 8 total, 8 up, 8 in 2024-04-26T04:50:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:16 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:17.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:16 smithi005 ceph-mon[36961]: osdmap e276: 8 total, 8 up, 8 in 2024-04-26T04:50:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:16 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:16 smithi005 ceph-mon[91801]: osdmap e276: 8 total, 8 up, 8 in 2024-04-26T04:50:17.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:17 smithi111 ceph-mon[30255]: pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:17 smithi111 ceph-mon[30255]: osdmap e277: 8 total, 8 up, 8 in 2024-04-26T04:50:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:17 smithi111 ceph-mon[30255]: osdmap e278: 8 total, 8 up, 8 in 2024-04-26T04:50:18.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:17 smithi005 ceph-mon[91801]: pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:17 smithi005 ceph-mon[91801]: osdmap e277: 8 total, 8 up, 8 in 2024-04-26T04:50:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:17 smithi005 ceph-mon[91801]: osdmap e278: 8 total, 8 up, 8 in 2024-04-26T04:50:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:17 smithi005 ceph-mon[36961]: pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:17 smithi005 ceph-mon[36961]: osdmap e277: 8 total, 8 up, 8 in 2024-04-26T04:50:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:17 smithi005 ceph-mon[36961]: osdmap e278: 8 total, 8 up, 8 in 2024-04-26T04:50:18.617 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:18.687 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:19.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:50:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:50:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:50:19.811 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:19.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:19 smithi005 ceph-mon[91801]: osdmap e279: 8 total, 8 up, 8 in 2024-04-26T04:50:19.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:19 smithi005 ceph-mon[91801]: pgmap v31: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 395 KiB/s rd, 29 KiB/s wr, 508 op/s 2024-04-26T04:50:19.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:19 smithi005 ceph-mon[36961]: osdmap e279: 8 total, 8 up, 8 in 2024-04-26T04:50:19.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:19 smithi005 ceph-mon[36961]: pgmap v31: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 395 KiB/s rd, 29 KiB/s wr, 508 op/s 2024-04-26T04:50:20.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:19 smithi111 ceph-mon[30255]: osdmap e279: 8 total, 8 up, 8 in 2024-04-26T04:50:20.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:19 smithi111 ceph-mon[30255]: pgmap v31: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 395 KiB/s rd, 29 KiB/s wr, 508 op/s 2024-04-26T04:50:20.743 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV1ParentAbort (3629 ms) 2024-04-26T04:50:20.743 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-04-26T04:50:20.777 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:20.787 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:20.796 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:20.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:20 smithi005 ceph-mon[91801]: osdmap e280: 8 total, 8 up, 8 in 2024-04-26T04:50:20.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:20 smithi005 ceph-mon[36961]: osdmap e280: 8 total, 8 up, 8 in 2024-04-26T04:50:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:20 smithi111 ceph-mon[30255]: osdmap e280: 8 total, 8 up, 8 in 2024-04-26T04:50:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:21 smithi005 ceph-mon[91801]: osdmap e281: 8 total, 8 up, 8 in 2024-04-26T04:50:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:21 smithi005 ceph-mon[91801]: pgmap v34: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 396 KiB/s rd, 29 KiB/s wr, 510 op/s 2024-04-26T04:50:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:21 smithi005 ceph-mon[36961]: osdmap e281: 8 total, 8 up, 8 in 2024-04-26T04:50:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:21 smithi005 ceph-mon[36961]: pgmap v34: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 396 KiB/s rd, 29 KiB/s wr, 510 op/s 2024-04-26T04:50:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:21 smithi111 ceph-mon[30255]: osdmap e281: 8 total, 8 up, 8 in 2024-04-26T04:50:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:21 smithi111 ceph-mon[30255]: pgmap v34: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 396 KiB/s rd, 29 KiB/s wr, 510 op/s 2024-04-26T04:50:22.639 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:22.700 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:22 smithi005 ceph-mon[36961]: osdmap e282: 8 total, 8 up, 8 in 2024-04-26T04:50:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:22 smithi005 ceph-mon[91801]: osdmap e282: 8 total, 8 up, 8 in 2024-04-26T04:50:22.997 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:22 smithi111 ceph-mon[30255]: osdmap e282: 8 total, 8 up, 8 in 2024-04-26T04:50:22.997 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:22 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-26T04:50:22.997 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:22 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:22.997 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:22 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:23.253 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:22 smithi111 podman[84321]: 2024-04-26 04:50:22.898761432 +0000 UTC m=+0.017179962 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:50:23.253 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 podman[84321]: 2024-04-26 04:50:23.024938315 +0000 UTC m=+0.143356831 container create 13f47dd5aa15393744213c3dd187cd43a09c19f51aefc09204885808f5037ce4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.buildah.version=1.32.0, release=239.1651231664, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.expose-services=, distribution-scope=public, name=ubi8, architecture=x86_64) 2024-04-26T04:50:23.253 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 podman[84321]: 2024-04-26 04:50:23.252564876 +0000 UTC m=+0.370983391 container init 13f47dd5aa15393744213c3dd187cd43a09c19f51aefc09204885808f5037ce4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, description=Ceph Grafana Container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-type=git) 2024-04-26T04:50:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:50:23] "GET /metrics HTTP/1.1" 200 38846 "" "Prometheus/2.43.0" 2024-04-26T04:50:23.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 podman[84321]: 2024-04-26 04:50:23.25558951 +0000 UTC m=+0.374008036 container start 13f47dd5aa15393744213c3dd187cd43a09c19f51aefc09204885808f5037ce4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, name=ubi8, architecture=x86_64, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , distribution-scope=public) 2024-04-26T04:50:23.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 bash[84321]: 13f47dd5aa15393744213c3dd187cd43a09c19f51aefc09204885808f5037ce4 2024-04-26T04:50:23.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:23.519 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[84335]: logger=settings t=2024-04-26T04:50:23.403099117Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:50:23.519 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:23 smithi111 podman[84352]: 2024-04-26 04:50:23.428842016 +0000 UTC m=+0.018782309 container died 13f47dd5aa15393744213c3dd187cd43a09c19f51aefc09204885808f5037ce4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, name=ubi8, io.buildah.version=1.32.0, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.openshift.expose-services=, io.openshift.tags=base rhel8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:50:23.804 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:23.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:23 smithi005 ceph-mon[91801]: osdmap e283: 8 total, 8 up, 8 in 2024-04-26T04:50:23.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:23 smithi005 ceph-mon[91801]: pgmap v37: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 261 KiB/s rd, 17 KiB/s wr, 336 op/s 2024-04-26T04:50:23.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:23 smithi005 ceph-mon[36961]: osdmap e283: 8 total, 8 up, 8 in 2024-04-26T04:50:23.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:23 smithi005 ceph-mon[36961]: pgmap v37: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 261 KiB/s rd, 17 KiB/s wr, 336 op/s 2024-04-26T04:50:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:23 smithi111 ceph-mon[30255]: osdmap e283: 8 total, 8 up, 8 in 2024-04-26T04:50:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:23 smithi111 ceph-mon[30255]: pgmap v37: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 261 KiB/s rd, 17 KiB/s wr, 336 op/s 2024-04-26T04:50:24.316 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:24 smithi111 podman[84352]: 2024-04-26 04:50:24.090368079 +0000 UTC m=+0.680308354 container remove 13f47dd5aa15393744213c3dd187cd43a09c19f51aefc09204885808f5037ce4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, release=239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:50:24.316 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:24 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:50:24.631 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:24 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:50:24.752 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4009 ms) 2024-04-26T04:50:24.752 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-04-26T04:50:24.781 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:24.791 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:24.799 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:24.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:24 smithi005 ceph-mon[91801]: osdmap e284: 8 total, 8 up, 8 in 2024-04-26T04:50:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:24 smithi005 ceph-mon[36961]: osdmap e284: 8 total, 8 up, 8 in 2024-04-26T04:50:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:24 smithi111 ceph-mon[30255]: osdmap e284: 8 total, 8 up, 8 in 2024-04-26T04:50:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:25 smithi005 ceph-mon[91801]: osdmap e285: 8 total, 8 up, 8 in 2024-04-26T04:50:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:25 smithi005 ceph-mon[91801]: pgmap v40: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 261 KiB/s rd, 17 KiB/s wr, 336 op/s 2024-04-26T04:50:25.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:25 smithi005 ceph-mon[36961]: osdmap e285: 8 total, 8 up, 8 in 2024-04-26T04:50:25.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:25 smithi005 ceph-mon[36961]: pgmap v40: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 261 KiB/s rd, 17 KiB/s wr, 336 op/s 2024-04-26T04:50:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:25 smithi111 ceph-mon[30255]: osdmap e285: 8 total, 8 up, 8 in 2024-04-26T04:50:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:25 smithi111 ceph-mon[30255]: pgmap v40: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 261 KiB/s rd, 17 KiB/s wr, 336 op/s 2024-04-26T04:50:26.662 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:26.721 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:26.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:26 smithi005 ceph-mon[91801]: osdmap e286: 8 total, 8 up, 8 in 2024-04-26T04:50:26.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:26 smithi005 ceph-mon[36961]: osdmap e286: 8 total, 8 up, 8 in 2024-04-26T04:50:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:26 smithi111 ceph-mon[30255]: osdmap e286: 8 total, 8 up, 8 in 2024-04-26T04:50:27.802 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:27.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:27 smithi005 ceph-mon[91801]: osdmap e287: 8 total, 8 up, 8 in 2024-04-26T04:50:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:27 smithi005 ceph-mon[91801]: pgmap v43: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:27 smithi005 ceph-mon[91801]: osdmap e288: 8 total, 8 up, 8 in 2024-04-26T04:50:27.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:27 smithi005 ceph-mon[36961]: osdmap e287: 8 total, 8 up, 8 in 2024-04-26T04:50:27.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:27 smithi005 ceph-mon[36961]: pgmap v43: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:27.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:27 smithi005 ceph-mon[36961]: osdmap e288: 8 total, 8 up, 8 in 2024-04-26T04:50:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:27 smithi111 ceph-mon[30255]: osdmap e287: 8 total, 8 up, 8 in 2024-04-26T04:50:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:27 smithi111 ceph-mon[30255]: pgmap v43: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:28.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:27 smithi111 ceph-mon[30255]: osdmap e288: 8 total, 8 up, 8 in 2024-04-26T04:50:28.788 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4036 ms) 2024-04-26T04:50:28.788 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-04-26T04:50:28.820 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:28.831 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:28.840 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:29.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:50:28 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:50:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:50:29.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:29 smithi005 ceph-mon[91801]: osdmap e289: 8 total, 8 up, 8 in 2024-04-26T04:50:29.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:29 smithi005 ceph-mon[91801]: pgmap v46: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 322 KiB/s rd, 25 KiB/s wr, 419 op/s 2024-04-26T04:50:29.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:29 smithi005 ceph-mon[36961]: osdmap e289: 8 total, 8 up, 8 in 2024-04-26T04:50:29.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:29 smithi005 ceph-mon[36961]: pgmap v46: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 322 KiB/s rd, 25 KiB/s wr, 419 op/s 2024-04-26T04:50:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:29 smithi111 ceph-mon[30255]: osdmap e289: 8 total, 8 up, 8 in 2024-04-26T04:50:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:29 smithi111 ceph-mon[30255]: pgmap v46: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 322 KiB/s rd, 25 KiB/s wr, 419 op/s 2024-04-26T04:50:30.684 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:30.751 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:30.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:30 smithi111 ceph-mon[30255]: osdmap e290: 8 total, 8 up, 8 in 2024-04-26T04:50:30.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:30 smithi005 ceph-mon[91801]: osdmap e290: 8 total, 8 up, 8 in 2024-04-26T04:50:30.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:30 smithi005 ceph-mon[36961]: osdmap e290: 8 total, 8 up, 8 in 2024-04-26T04:50:31.869 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:31.932 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:31 smithi111 ceph-mon[30255]: osdmap e291: 8 total, 8 up, 8 in 2024-04-26T04:50:31.932 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:31 smithi111 ceph-mon[30255]: pgmap v49: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 322 KiB/s rd, 25 KiB/s wr, 419 op/s 2024-04-26T04:50:31.932 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:31 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:31.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:31 smithi005 ceph-mon[91801]: osdmap e291: 8 total, 8 up, 8 in 2024-04-26T04:50:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:31 smithi005 ceph-mon[91801]: pgmap v49: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 322 KiB/s rd, 25 KiB/s wr, 419 op/s 2024-04-26T04:50:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:31 smithi005 ceph-mon[36961]: osdmap e291: 8 total, 8 up, 8 in 2024-04-26T04:50:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:31 smithi005 ceph-mon[36961]: pgmap v49: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 322 KiB/s rd, 25 KiB/s wr, 419 op/s 2024-04-26T04:50:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:31 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:32.793 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4005 ms) 2024-04-26T04:50:32.794 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-04-26T04:50:32.824 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:32.834 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:32.843 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:32.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:32 smithi005 ceph-mon[91801]: osdmap e292: 8 total, 8 up, 8 in 2024-04-26T04:50:32.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:32 smithi005 ceph-mon[36961]: osdmap e292: 8 total, 8 up, 8 in 2024-04-26T04:50:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:32 smithi111 ceph-mon[30255]: osdmap e292: 8 total, 8 up, 8 in 2024-04-26T04:50:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:50:33] "GET /metrics HTTP/1.1" 200 38848 "" "Prometheus/2.43.0" 2024-04-26T04:50:33.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:33 smithi005 ceph-mon[91801]: osdmap e293: 8 total, 8 up, 8 in 2024-04-26T04:50:33.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:33 smithi005 ceph-mon[91801]: pgmap v52: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 410 KiB/s rd, 21 KiB/s wr, 514 op/s 2024-04-26T04:50:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:33 smithi005 ceph-mon[36961]: osdmap e293: 8 total, 8 up, 8 in 2024-04-26T04:50:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:33 smithi005 ceph-mon[36961]: pgmap v52: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 410 KiB/s rd, 21 KiB/s wr, 514 op/s 2024-04-26T04:50:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:33 smithi111 ceph-mon[30255]: osdmap e293: 8 total, 8 up, 8 in 2024-04-26T04:50:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:33 smithi111 ceph-mon[30255]: pgmap v52: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 410 KiB/s rd, 21 KiB/s wr, 514 op/s 2024-04-26T04:50:34.701 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:34.757 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:34.783 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:34 smithi111 ceph-mon[30255]: osdmap e294: 8 total, 8 up, 8 in 2024-04-26T04:50:34.783 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:34 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-26T04:50:34.783 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:34 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:34.783 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:34 smithi111 systemd[1]: Starting Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:34.783 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:34 smithi111 podman[84456]: 2024-04-26 04:50:34.656281516 +0000 UTC m=+0.018179531 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:50:34.784 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:34 smithi111 podman[84456]: 2024-04-26 04:50:34.783220471 +0000 UTC m=+0.145118471 container create 567f60e979d8c938d61b60cbc2542a526ea8cc8d50e32a770cfef0dbb97c521b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , description=Ceph Grafana Container, version=8.5, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:50:34.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:34 smithi005 ceph-mon[91801]: osdmap e294: 8 total, 8 up, 8 in 2024-04-26T04:50:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:34 smithi005 ceph-mon[36961]: osdmap e294: 8 total, 8 up, 8 in 2024-04-26T04:50:35.149 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 podman[84456]: 2024-04-26 04:50:35.027169382 +0000 UTC m=+0.389067384 container init 567f60e979d8c938d61b60cbc2542a526ea8cc8d50e32a770cfef0dbb97c521b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, name=ubi8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , release=239.1651231664, version=8.5, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:50:35.149 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 podman[84456]: 2024-04-26 04:50:35.029780968 +0000 UTC m=+0.391678971 container start 567f60e979d8c938d61b60cbc2542a526ea8cc8d50e32a770cfef0dbb97c521b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, name=ubi8, architecture=x86_64) 2024-04-26T04:50:35.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 bash[84456]: 567f60e979d8c938d61b60cbc2542a526ea8cc8d50e32a770cfef0dbb97c521b 2024-04-26T04:50:35.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 systemd[1]: Started Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:35.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a[84471]: logger=settings t=2024-04-26T04:50:35.181393779Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:50:35.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 podman[84488]: 2024-04-26 04:50:35.206827341 +0000 UTC m=+0.018182773 container died 567f60e979d8c938d61b60cbc2542a526ea8cc8d50e32a770cfef0dbb97c521b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, release=239.1651231664, name=ubi8, version=8.5, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.32.0) 2024-04-26T04:50:35.899 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:35.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:35 smithi005 ceph-mon[36961]: osdmap e295: 8 total, 8 up, 8 in 2024-04-26T04:50:35.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:35 smithi005 ceph-mon[36961]: pgmap v55: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 410 KiB/s rd, 21 KiB/s wr, 514 op/s 2024-04-26T04:50:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:35 smithi005 ceph-mon[91801]: osdmap e295: 8 total, 8 up, 8 in 2024-04-26T04:50:35.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:35 smithi005 ceph-mon[91801]: pgmap v55: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 410 KiB/s rd, 21 KiB/s wr, 514 op/s 2024-04-26T04:50:36.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:35 smithi111 ceph-mon[30255]: osdmap e295: 8 total, 8 up, 8 in 2024-04-26T04:50:36.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:35 smithi111 ceph-mon[30255]: pgmap v55: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 410 KiB/s rd, 21 KiB/s wr, 514 op/s 2024-04-26T04:50:36.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 podman[84488]: 2024-04-26 04:50:35.898657609 +0000 UTC m=+0.710013087 container remove 567f60e979d8c938d61b60cbc2542a526ea8cc8d50e32a770cfef0dbb97c521b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, name=ubi8, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, version=8.5, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:50:36.018 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:35 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:50:36.518 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:36 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:50:36.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4031 ms) 2024-04-26T04:50:36.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-04-26T04:50:36.855 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:36.865 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:36.874 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:36.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:36 smithi005 ceph-mon[91801]: osdmap e296: 8 total, 8 up, 8 in 2024-04-26T04:50:36.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:36 smithi005 ceph-mon[36961]: osdmap e296: 8 total, 8 up, 8 in 2024-04-26T04:50:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:36 smithi111 ceph-mon[30255]: osdmap e296: 8 total, 8 up, 8 in 2024-04-26T04:50:37.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:37 smithi005 ceph-mon[91801]: osdmap e297: 8 total, 8 up, 8 in 2024-04-26T04:50:37.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:37 smithi005 ceph-mon[91801]: pgmap v58: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:37.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:37 smithi005 ceph-mon[91801]: osdmap e298: 8 total, 8 up, 8 in 2024-04-26T04:50:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:37 smithi005 ceph-mon[36961]: osdmap e297: 8 total, 8 up, 8 in 2024-04-26T04:50:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:37 smithi005 ceph-mon[36961]: pgmap v58: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:37 smithi005 ceph-mon[36961]: osdmap e298: 8 total, 8 up, 8 in 2024-04-26T04:50:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:37 smithi111 ceph-mon[30255]: osdmap e297: 8 total, 8 up, 8 in 2024-04-26T04:50:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:37 smithi111 ceph-mon[30255]: pgmap v58: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:37 smithi111 ceph-mon[30255]: osdmap e298: 8 total, 8 up, 8 in 2024-04-26T04:50:38.621 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:38.681 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:39.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:50:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:50:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:50:39.805 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:39.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:39 smithi005 ceph-mon[36961]: osdmap e299: 8 total, 8 up, 8 in 2024-04-26T04:50:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:39 smithi005 ceph-mon[36961]: pgmap v61: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 22 KiB/s wr, 410 op/s 2024-04-26T04:50:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:39 smithi005 ceph-mon[91801]: osdmap e299: 8 total, 8 up, 8 in 2024-04-26T04:50:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:39 smithi005 ceph-mon[91801]: pgmap v61: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 22 KiB/s wr, 410 op/s 2024-04-26T04:50:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:39 smithi111 ceph-mon[30255]: osdmap e299: 8 total, 8 up, 8 in 2024-04-26T04:50:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:39 smithi111 ceph-mon[30255]: pgmap v61: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 22 KiB/s wr, 410 op/s 2024-04-26T04:50:40.752 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (3928 ms) 2024-04-26T04:50:40.752 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-04-26T04:50:40.787 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:40.798 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:40.806 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:40.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:40 smithi005 ceph-mon[91801]: osdmap e300: 8 total, 8 up, 8 in 2024-04-26T04:50:40.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:40 smithi005 ceph-mon[36961]: osdmap e300: 8 total, 8 up, 8 in 2024-04-26T04:50:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:40 smithi111 ceph-mon[30255]: osdmap e300: 8 total, 8 up, 8 in 2024-04-26T04:50:41.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:41 smithi005 ceph-mon[36961]: osdmap e301: 8 total, 8 up, 8 in 2024-04-26T04:50:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:41 smithi005 ceph-mon[36961]: pgmap v64: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 22 KiB/s wr, 410 op/s 2024-04-26T04:50:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:41 smithi005 ceph-mon[91801]: osdmap e301: 8 total, 8 up, 8 in 2024-04-26T04:50:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:41 smithi005 ceph-mon[91801]: pgmap v64: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 22 KiB/s wr, 410 op/s 2024-04-26T04:50:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:41 smithi111 ceph-mon[30255]: osdmap e301: 8 total, 8 up, 8 in 2024-04-26T04:50:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:41 smithi111 ceph-mon[30255]: pgmap v64: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 22 KiB/s wr, 410 op/s 2024-04-26T04:50:42.640 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:42.703 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:42.880 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:42 smithi005 ceph-mon[91801]: osdmap e302: 8 total, 8 up, 8 in 2024-04-26T04:50:42.880 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:42 smithi005 ceph-mon[36961]: osdmap e302: 8 total, 8 up, 8 in 2024-04-26T04:50:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:42 smithi111 ceph-mon[30255]: osdmap e302: 8 total, 8 up, 8 in 2024-04-26T04:50:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:50:43] "GET /metrics HTTP/1.1" 200 38848 "" "Prometheus/2.43.0" 2024-04-26T04:50:43.831 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:43.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:43 smithi005 ceph-mon[36961]: osdmap e303: 8 total, 8 up, 8 in 2024-04-26T04:50:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:43 smithi005 ceph-mon[36961]: pgmap v67: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 249 KiB/s rd, 21 KiB/s wr, 323 op/s 2024-04-26T04:50:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:43 smithi005 ceph-mon[91801]: osdmap e303: 8 total, 8 up, 8 in 2024-04-26T04:50:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:43 smithi005 ceph-mon[91801]: pgmap v67: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 249 KiB/s rd, 21 KiB/s wr, 323 op/s 2024-04-26T04:50:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:43 smithi111 ceph-mon[30255]: osdmap e303: 8 total, 8 up, 8 in 2024-04-26T04:50:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:43 smithi111 ceph-mon[30255]: pgmap v67: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 249 KiB/s rd, 21 KiB/s wr, 323 op/s 2024-04-26T04:50:44.776 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4024 ms) 2024-04-26T04:50:44.776 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-04-26T04:50:44.812 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:44.824 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 0~10 2024-04-26T04:50:44.833 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:44.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:44 smithi005 ceph-mon[91801]: osdmap e304: 8 total, 8 up, 8 in 2024-04-26T04:50:44.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:44 smithi005 ceph-mon[36961]: osdmap e304: 8 total, 8 up, 8 in 2024-04-26T04:50:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:44 smithi111 ceph-mon[30255]: osdmap e304: 8 total, 8 up, 8 in 2024-04-26T04:50:45.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:45 smithi005 ceph-mon[91801]: osdmap e305: 8 total, 8 up, 8 in 2024-04-26T04:50:45.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:45 smithi005 ceph-mon[91801]: pgmap v70: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 249 KiB/s rd, 21 KiB/s wr, 323 op/s 2024-04-26T04:50:45.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:45 smithi005 ceph-mon[36961]: osdmap e305: 8 total, 8 up, 8 in 2024-04-26T04:50:45.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:45 smithi005 ceph-mon[36961]: pgmap v70: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 249 KiB/s rd, 21 KiB/s wr, 323 op/s 2024-04-26T04:50:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:45 smithi111 ceph-mon[30255]: osdmap e305: 8 total, 8 up, 8 in 2024-04-26T04:50:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:45 smithi111 ceph-mon[30255]: pgmap v70: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 249 KiB/s rd, 21 KiB/s wr, 323 op/s 2024-04-26T04:50:46.648 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:46 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-26T04:50:46.648 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:46 smithi111 systemd[1]: Stopped Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:46.648 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:46 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-26T04:50:46.648 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:46 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:50:46.648 INFO:journalctl@ceph.grafana.a.smithi111.stdout:Apr 26 04:50:46 smithi111 systemd[1]: Failed to start Ceph grafana.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:46.668 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:46.730 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:50:46.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:46 smithi005 ceph-mon[91801]: osdmap e306: 8 total, 8 up, 8 in 2024-04-26T04:50:46.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:46 smithi005 ceph-mon[36961]: osdmap e306: 8 total, 8 up, 8 in 2024-04-26T04:50:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:46 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:46 smithi111 ceph-mon[30255]: osdmap e306: 8 total, 8 up, 8 in 2024-04-26T04:50:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:46 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:47.798 INFO:tasks.workunit.client.0.smithi005.stdout:migration_abort 2024-04-26T04:50:47.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:47 smithi005 ceph-mon[36961]: osdmap e307: 8 total, 8 up, 8 in 2024-04-26T04:50:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:47 smithi005 ceph-mon[36961]: pgmap v73: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:47 smithi005 ceph-mon[36961]: osdmap e308: 8 total, 8 up, 8 in 2024-04-26T04:50:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:47 smithi005 ceph-mon[91801]: osdmap e307: 8 total, 8 up, 8 in 2024-04-26T04:50:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:47 smithi005 ceph-mon[91801]: pgmap v73: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:47 smithi005 ceph-mon[91801]: osdmap e308: 8 total, 8 up, 8 in 2024-04-26T04:50:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:47 smithi111 ceph-mon[30255]: osdmap e307: 8 total, 8 up, 8 in 2024-04-26T04:50:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:47 smithi111 ceph-mon[30255]: pgmap v73: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:47 smithi111 ceph-mon[30255]: osdmap e308: 8 total, 8 up, 8 in 2024-04-26T04:50:48.793 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (4016 ms) 2024-04-26T04:50:48.793 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-04-26T04:50:48.824 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:48.835 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 76732260~7153820 2024-04-26T04:50:48.884 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 3772681~4641420 2024-04-26T04:50:48.884 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 53787349~8318636 2024-04-26T04:50:48.930 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 79878376~4007704 2024-04-26T04:50:48.932 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 32771406~5780906 2024-04-26T04:50:48.964 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 32541491~8073768 2024-04-26T04:50:48.972 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 47279835~8064496 2024-04-26T04:50:49.011 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 36346491~7292704 2024-04-26T04:50:49.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:50:48 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:50:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:50:49.018 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap0 2024-04-26T04:50:49.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:49 smithi005 ceph-mon[36961]: osdmap e309: 8 total, 8 up, 8 in 2024-04-26T04:50:49.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:49 smithi005 ceph-mon[36961]: pgmap v76: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 321 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:50:49.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:49 smithi005 ceph-mon[91801]: osdmap e309: 8 total, 8 up, 8 in 2024-04-26T04:50:49.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:49 smithi005 ceph-mon[91801]: pgmap v76: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 321 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:50:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:49 smithi111 ceph-mon[30255]: osdmap e309: 8 total, 8 up, 8 in 2024-04-26T04:50:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:49 smithi111 ceph-mon[30255]: pgmap v76: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 321 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:50:50.678 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap0 2024-04-26T04:50:50.693 INFO:tasks.workunit.client.0.smithi005.stdout:clone image159 -> image162 2024-04-26T04:50:50.789 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 128359864 2024-04-26T04:50:50.800 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 65806089~1380485 2024-04-26T04:50:50.821 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 10464652~5083280 2024-04-26T04:50:50.827 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 46582524~2847328 2024-04-26T04:50:50.872 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 73325407~8264151 2024-04-26T04:50:50.879 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 28397596~4254320 2024-04-26T04:50:50.905 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 54320510~3133768 2024-04-26T04:50:50.961 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 75293276~6403712 2024-04-26T04:50:50.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:50 smithi005 ceph-mon[91801]: osdmap e310: 8 total, 8 up, 8 in 2024-04-26T04:50:50.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:50 smithi005 ceph-mon[36961]: osdmap e310: 8 total, 8 up, 8 in 2024-04-26T04:50:50.993 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 66224290~7833906 2024-04-26T04:50:50.996 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:50 smithi111 ceph-mon[30255]: osdmap e310: 8 total, 8 up, 8 in 2024-04-26T04:50:51.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:51 smithi005 ceph-mon[36961]: osdmap e311: 8 total, 8 up, 8 in 2024-04-26T04:50:51.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:51 smithi005 ceph-mon[36961]: pgmap v79: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 321 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:50:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:51 smithi005 ceph-mon[91801]: osdmap e311: 8 total, 8 up, 8 in 2024-04-26T04:50:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:51 smithi005 ceph-mon[91801]: pgmap v79: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 321 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:50:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:51 smithi111 ceph-mon[30255]: osdmap e311: 8 total, 8 up, 8 in 2024-04-26T04:50:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:51 smithi111 ceph-mon[30255]: pgmap v79: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 321 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:50:52.696 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:50:52.708 INFO:tasks.workunit.client.0.smithi005.stdout:clone image162 -> image164 2024-04-26T04:50:52.831 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 165179042 2024-04-26T04:50:52.843 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 19663631~7235606 2024-04-26T04:50:52.898 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 155478694~5012464 2024-04-26T04:50:52.898 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 49989867~7668964 2024-04-26T04:50:52.957 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 159551868~3868953 2024-04-26T04:50:52.957 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 93843079~7480412 2024-04-26T04:50:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:52 smithi005 ceph-mon[91801]: osdmap e312: 8 total, 8 up, 8 in 2024-04-26T04:50:52.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:52 smithi005 ceph-mon[36961]: osdmap e312: 8 total, 8 up, 8 in 2024-04-26T04:50:52.993 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 157556838~7622204 2024-04-26T04:50:52.993 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 14115651~2183521 2024-04-26T04:50:53.012 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 24771881~6705828 2024-04-26T04:50:53.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:52 smithi111 ceph-mon[30255]: osdmap e312: 8 total, 8 up, 8 in 2024-04-26T04:50:53.034 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:50:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:50:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:50:53] "GET /metrics HTTP/1.1" 200 38847 "" "Prometheus/2.43.0" 2024-04-26T04:50:53.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:53 smithi005 ceph-mon[91801]: osdmap e313: 8 total, 8 up, 8 in 2024-04-26T04:50:53.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:53 smithi005 ceph-mon[91801]: pgmap v82: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s rd, 24 MiB/s wr, 469 op/s 2024-04-26T04:50:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:53 smithi005 ceph-mon[36961]: osdmap e313: 8 total, 8 up, 8 in 2024-04-26T04:50:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:53 smithi005 ceph-mon[36961]: pgmap v82: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s rd, 24 MiB/s wr, 469 op/s 2024-04-26T04:50:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:53 smithi111 ceph-mon[30255]: osdmap e313: 8 total, 8 up, 8 in 2024-04-26T04:50:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:53 smithi111 ceph-mon[30255]: pgmap v82: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s rd, 24 MiB/s wr, 469 op/s 2024-04-26T04:50:54.706 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap2 2024-04-26T04:50:54.718 INFO:tasks.workunit.client.0.smithi005.stdout:clone image164 -> image166 2024-04-26T04:50:54.872 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 161132351~2683972 2024-04-26T04:50:54.904 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 45251514~698401 2024-04-26T04:50:54.910 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 65204261~5491821 2024-04-26T04:50:54.954 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 81972586~8337577 2024-04-26T04:50:54.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:54 smithi005 ceph-mon[91801]: osdmap e314: 8 total, 8 up, 8 in 2024-04-26T04:50:54.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:54 smithi005 ceph-mon[36961]: osdmap e314: 8 total, 8 up, 8 in 2024-04-26T04:50:54.981 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 85013581~3336185 2024-04-26T04:50:55.006 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 92107128~3420707 2024-04-26T04:50:55.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:54 smithi111 ceph-mon[30255]: osdmap e314: 8 total, 8 up, 8 in 2024-04-26T04:50:55.026 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 118489709~7532245 2024-04-26T04:50:55.067 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 49457616~6530481 2024-04-26T04:50:55.115 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:50:55.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:55 smithi005 ceph-mon[91801]: osdmap e315: 8 total, 8 up, 8 in 2024-04-26T04:50:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:55 smithi005 ceph-mon[91801]: pgmap v85: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s rd, 24 MiB/s wr, 469 op/s 2024-04-26T04:50:55.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:55 smithi005 ceph-mon[36961]: osdmap e315: 8 total, 8 up, 8 in 2024-04-26T04:50:55.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:55 smithi005 ceph-mon[36961]: pgmap v85: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s rd, 24 MiB/s wr, 469 op/s 2024-04-26T04:50:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:55 smithi111 ceph-mon[30255]: osdmap e315: 8 total, 8 up, 8 in 2024-04-26T04:50:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:55 smithi111 ceph-mon[30255]: pgmap v85: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.9 MiB/s rd, 24 MiB/s wr, 469 op/s 2024-04-26T04:50:56.720 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap3 2024-04-26T04:50:56.732 INFO:tasks.workunit.client.0.smithi005.stdout:clone image166 -> image168 2024-04-26T04:50:56.917 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 183177461 2024-04-26T04:50:56.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:56 smithi005 ceph-mon[91801]: osdmap e316: 8 total, 8 up, 8 in 2024-04-26T04:50:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:56 smithi005 ceph-mon[36961]: osdmap e316: 8 total, 8 up, 8 in 2024-04-26T04:50:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:56 smithi111 ceph-mon[30255]: osdmap e316: 8 total, 8 up, 8 in 2024-04-26T04:50:57.151 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.StressNoMigrate (8359 ms) 2024-04-26T04:50:57.151 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.Stress 2024-04-26T04:50:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:50:57.196 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 79217263~4668817 2024-04-26T04:50:57.234 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 13860460~7936259 2024-04-26T04:50:57.234 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 1170199~3228885 2024-04-26T04:50:57.263 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 56694459~7655373 2024-04-26T04:50:57.263 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 52406842~7876027 2024-04-26T04:50:57.305 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 79817580~3955456 2024-04-26T04:50:57.307 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 79126914~4759166 2024-04-26T04:50:57.334 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 78867571~5018509 2024-04-26T04:50:57.340 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap0 2024-04-26T04:50:57.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:57 smithi005 ceph-mon[91801]: osdmap e317: 8 total, 8 up, 8 in 2024-04-26T04:50:57.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:57 smithi005 ceph-mon[91801]: pgmap v88: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:57 smithi005 ceph-mon[91801]: osdmap e318: 8 total, 8 up, 8 in 2024-04-26T04:50:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:57 smithi005 ceph-mon[36961]: osdmap e317: 8 total, 8 up, 8 in 2024-04-26T04:50:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:57 smithi005 ceph-mon[36961]: pgmap v88: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:57 smithi005 ceph-mon[36961]: osdmap e318: 8 total, 8 up, 8 in 2024-04-26T04:50:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:57 smithi111 ceph-mon[30255]: osdmap e317: 8 total, 8 up, 8 in 2024-04-26T04:50:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:57 smithi111 ceph-mon[30255]: pgmap v88: 137 pgs: 137 active+clean; 1014 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:50:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:57 smithi111 ceph-mon[30255]: osdmap e318: 8 total, 8 up, 8 in 2024-04-26T04:50:58.626 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap0 2024-04-26T04:50:58.638 INFO:tasks.workunit.client.0.smithi005.stdout:clone image169 -> image172 2024-04-26T04:50:58.737 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 16296581~6642000 2024-04-26T04:50:58.783 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16767363~3856973 2024-04-26T04:50:58.786 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 1705549~2940072 2024-04-26T04:50:58.824 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 55539425~7522392 2024-04-26T04:50:58.847 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 62010142~8236030 2024-04-26T04:50:58.888 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 30824083~6007640 2024-04-26T04:50:58.894 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 10676457~3745733 2024-04-26T04:50:58.915 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 36847816~1264940 2024-04-26T04:50:58.920 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:50:59.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:50:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:50:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:50:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:59 smithi005 ceph-mon[91801]: osdmap e319: 8 total, 8 up, 8 in 2024-04-26T04:50:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:50:59 smithi005 ceph-mon[91801]: pgmap v91: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 38 MiB/s wr, 807 op/s 2024-04-26T04:50:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:59 smithi005 ceph-mon[36961]: osdmap e319: 8 total, 8 up, 8 in 2024-04-26T04:50:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:50:59 smithi005 ceph-mon[36961]: pgmap v91: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 38 MiB/s wr, 807 op/s 2024-04-26T04:51:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:59 smithi111 ceph-mon[30255]: osdmap e319: 8 total, 8 up, 8 in 2024-04-26T04:51:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:50:59 smithi111 ceph-mon[30255]: pgmap v91: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 38 MiB/s wr, 807 op/s 2024-04-26T04:51:00.633 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:51:00.649 INFO:tasks.workunit.client.0.smithi005.stdout:clone image172 -> image174 2024-04-26T04:51:00.785 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 57358115~4463583 2024-04-26T04:51:00.838 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 37505435~2468029 2024-04-26T04:51:00.844 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 6423485~713054 2024-04-26T04:51:00.856 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 30638044~8271111 2024-04-26T04:51:00.862 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 72009876~4348387 2024-04-26T04:51:00.891 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 32930934~3776643 2024-04-26T04:51:00.893 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 2030035~5421800 2024-04-26T04:51:00.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:00 smithi111 ceph-mon[30255]: osdmap e320: 8 total, 8 up, 8 in 2024-04-26T04:51:00.930 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 42833205~7284475 2024-04-26T04:51:00.937 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:51:00.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:00 smithi005 ceph-mon[91801]: osdmap e320: 8 total, 8 up, 8 in 2024-04-26T04:51:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:00 smithi005 ceph-mon[36961]: osdmap e320: 8 total, 8 up, 8 in 2024-04-26T04:51:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:01 smithi005 ceph-mon[91801]: osdmap e321: 8 total, 8 up, 8 in 2024-04-26T04:51:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:01 smithi005 ceph-mon[91801]: pgmap v94: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 38 MiB/s wr, 807 op/s 2024-04-26T04:51:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:01 smithi005 ceph-mon[36961]: osdmap e321: 8 total, 8 up, 8 in 2024-04-26T04:51:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:01 smithi005 ceph-mon[36961]: pgmap v94: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 38 MiB/s wr, 807 op/s 2024-04-26T04:51:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:01 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:01 smithi111 ceph-mon[30255]: osdmap e321: 8 total, 8 up, 8 in 2024-04-26T04:51:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:01 smithi111 ceph-mon[30255]: pgmap v94: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 38 MiB/s wr, 807 op/s 2024-04-26T04:51:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:02.654 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap2 2024-04-26T04:51:02.668 INFO:tasks.workunit.client.0.smithi005.stdout:clone image174 -> image176 2024-04-26T04:51:02.825 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 161979266 2024-04-26T04:51:02.836 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 124484228~1153967 2024-04-26T04:51:02.850 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 5045319~2443136 2024-04-26T04:51:02.868 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 71221763~4662748 2024-04-26T04:51:02.921 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 100428218~8189587 2024-04-26T04:51:02.921 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 91982021~977727 2024-04-26T04:51:02.931 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 13670311~6321692 2024-04-26T04:51:02.951 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 143577391~7557947 2024-04-26T04:51:02.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:02 smithi005 ceph-mon[91801]: osdmap e322: 8 total, 8 up, 8 in 2024-04-26T04:51:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:02 smithi005 ceph-mon[36961]: osdmap e322: 8 total, 8 up, 8 in 2024-04-26T04:51:02.995 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 133349574~7939582 2024-04-26T04:51:02.995 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:51:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:02 smithi111 ceph-mon[30255]: osdmap e322: 8 total, 8 up, 8 in 2024-04-26T04:51:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:51:03] "GET /metrics HTTP/1.1" 200 38843 "" "Prometheus/2.43.0" 2024-04-26T04:51:03.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:03 smithi005 ceph-mon[91801]: osdmap e323: 8 total, 8 up, 8 in 2024-04-26T04:51:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:03 smithi005 ceph-mon[91801]: pgmap v97: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 20 MiB/s wr, 407 op/s 2024-04-26T04:51:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:03 smithi005 ceph-mon[36961]: osdmap e323: 8 total, 8 up, 8 in 2024-04-26T04:51:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:03 smithi005 ceph-mon[36961]: pgmap v97: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 20 MiB/s wr, 407 op/s 2024-04-26T04:51:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:03 smithi111 ceph-mon[30255]: osdmap e323: 8 total, 8 up, 8 in 2024-04-26T04:51:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:03 smithi111 ceph-mon[30255]: pgmap v97: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 20 MiB/s wr, 407 op/s 2024-04-26T04:51:04.665 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap3 2024-04-26T04:51:04.680 INFO:tasks.workunit.client.0.smithi005.stdout:clone image176 -> image178 2024-04-26T04:51:04.867 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:51:04.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:04 smithi005 ceph-mon[91801]: osdmap e324: 8 total, 8 up, 8 in 2024-04-26T04:51:04.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:04 smithi005 ceph-mon[36961]: osdmap e324: 8 total, 8 up, 8 in 2024-04-26T04:51:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:04 smithi111 ceph-mon[30255]: osdmap e324: 8 total, 8 up, 8 in 2024-04-26T04:51:05.355 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:51:05.574 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:51:05.881 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.Stress (8730 ms) 2024-04-26T04:51:05.881 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.Stress2 2024-04-26T04:51:05.911 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:51:05.921 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 53087866~3002845 2024-04-26T04:51:05.945 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 7903857~1934239 2024-04-26T04:51:05.945 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 76808213~6532861 2024-04-26T04:51:05.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:05 smithi005 ceph-mon[91801]: osdmap e325: 8 total, 8 up, 8 in 2024-04-26T04:51:05.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:05 smithi005 ceph-mon[91801]: pgmap v100: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 20 MiB/s wr, 407 op/s 2024-04-26T04:51:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:05 smithi005 ceph-mon[36961]: osdmap e325: 8 total, 8 up, 8 in 2024-04-26T04:51:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:05 smithi005 ceph-mon[36961]: pgmap v100: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 20 MiB/s wr, 407 op/s 2024-04-26T04:51:05.981 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 33728721~4937049 2024-04-26T04:51:05.981 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 39381646~206608 2024-04-26T04:51:05.987 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 60762803~2194471 2024-04-26T04:51:05.987 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 62068694~34320 2024-04-26T04:51:05.992 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 27643280~697655 2024-04-26T04:51:05.992 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap0 2024-04-26T04:51:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:05 smithi111 ceph-mon[30255]: osdmap e325: 8 total, 8 up, 8 in 2024-04-26T04:51:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:05 smithi111 ceph-mon[30255]: pgmap v100: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 20 MiB/s wr, 407 op/s 2024-04-26T04:51:07.678 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 136407971 2024-04-26T04:51:07.693 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 87063745~7593271 2024-04-26T04:51:07.709 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 94201350~3167970 2024-04-26T04:51:07.731 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 79384146~1855477 2024-04-26T04:51:07.732 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 69543407~1468365 2024-04-26T04:51:07.732 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 114713762~8190498 2024-04-26T04:51:07.740 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 95116067~4099026 2024-04-26T04:51:07.740 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 61195014~7591552 2024-04-26T04:51:07.747 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 68718087~7710589 2024-04-26T04:51:07.797 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:51:07.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:07 smithi005 ceph-mon[36961]: osdmap e326: 8 total, 8 up, 8 in 2024-04-26T04:51:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:07 smithi005 ceph-mon[36961]: pgmap v102: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:51:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:07 smithi005 ceph-mon[91801]: osdmap e326: 8 total, 8 up, 8 in 2024-04-26T04:51:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:07 smithi005 ceph-mon[91801]: pgmap v102: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:51:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:07 smithi111 ceph-mon[30255]: osdmap e326: 8 total, 8 up, 8 in 2024-04-26T04:51:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:07 smithi111 ceph-mon[30255]: pgmap v102: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:51:08.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:08 smithi005 ceph-mon[91801]: osdmap e327: 8 total, 8 up, 8 in 2024-04-26T04:51:08.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:08 smithi005 ceph-mon[36961]: osdmap e327: 8 total, 8 up, 8 in 2024-04-26T04:51:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:08 smithi111 ceph-mon[30255]: osdmap e327: 8 total, 8 up, 8 in 2024-04-26T04:51:09.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:51:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:51:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:51:09.685 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap1 2024-04-26T04:51:09.701 INFO:tasks.workunit.client.0.smithi005.stdout:clone image179 -> image182 2024-04-26T04:51:09.808 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 63149315~3655242 2024-04-26T04:51:09.868 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 31713309~839330 2024-04-26T04:51:09.873 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 127593905~7368258 2024-04-26T04:51:09.921 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 25303670~1128168 2024-04-26T04:51:09.926 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 106086799~3579532 2024-04-26T04:51:09.947 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 32988668~1731773 2024-04-26T04:51:09.952 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 25729896~5795867 2024-04-26T04:51:09.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:09 smithi005 ceph-mon[36961]: osdmap e328: 8 total, 8 up, 8 in 2024-04-26T04:51:09.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:09 smithi005 ceph-mon[36961]: pgmap v105: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 43 MiB/s rd, 14 MiB/s wr, 1.28k op/s 2024-04-26T04:51:09.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:09 smithi005 ceph-mon[91801]: osdmap e328: 8 total, 8 up, 8 in 2024-04-26T04:51:09.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:09 smithi005 ceph-mon[91801]: pgmap v105: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 43 MiB/s rd, 14 MiB/s wr, 1.28k op/s 2024-04-26T04:51:09.987 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 69129609~4473721 2024-04-26T04:51:10.009 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:51:10.021 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:09 smithi111 ceph-mon[30255]: osdmap e328: 8 total, 8 up, 8 in 2024-04-26T04:51:10.022 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:09 smithi111 ceph-mon[30255]: pgmap v105: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 43 MiB/s rd, 14 MiB/s wr, 1.28k op/s 2024-04-26T04:51:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:10 smithi005 ceph-mon[91801]: osdmap e329: 8 total, 8 up, 8 in 2024-04-26T04:51:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:10 smithi005 ceph-mon[36961]: osdmap e329: 8 total, 8 up, 8 in 2024-04-26T04:51:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:10 smithi111 ceph-mon[30255]: osdmap e329: 8 total, 8 up, 8 in 2024-04-26T04:51:11.039 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:51:11.312 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:11.699 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap2 2024-04-26T04:51:11.711 INFO:tasks.workunit.client.0.smithi005.stdout:clone image182 -> image184 2024-04-26T04:51:11.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:11 smithi005 ceph-mon[91801]: osdmap e330: 8 total, 8 up, 8 in 2024-04-26T04:51:11.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:11 smithi005 ceph-mon[91801]: pgmap v108: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 15 MiB/s wr, 1.34k op/s 2024-04-26T04:51:11.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:11 smithi005 ceph-mon[36961]: osdmap e330: 8 total, 8 up, 8 in 2024-04-26T04:51:11.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:11 smithi005 ceph-mon[36961]: pgmap v108: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 15 MiB/s wr, 1.34k op/s 2024-04-26T04:51:11.831 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 95991864 2024-04-26T04:51:11.881 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 77689868~4703828 2024-04-26T04:51:11.943 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 1003775~628036 2024-04-26T04:51:11.949 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 39204259~466038 2024-04-26T04:51:11.959 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 21335579~106872 2024-04-26T04:51:11.965 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 52119329~5726368 2024-04-26T04:51:12.004 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 39913523~3932173 2024-04-26T04:51:12.010 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 77368776~2480308 2024-04-26T04:51:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:11 smithi111 ceph-mon[30255]: osdmap e330: 8 total, 8 up, 8 in 2024-04-26T04:51:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:11 smithi111 ceph-mon[30255]: pgmap v108: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 15 MiB/s wr, 1.34k op/s 2024-04-26T04:51:12.026 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 24268244~7198421 2024-04-26T04:51:12.043 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:51:12.840 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:12 smithi005 ceph-mon[91801]: osdmap e331: 8 total, 8 up, 8 in 2024-04-26T04:51:12.841 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:12 smithi005 ceph-mon[36961]: osdmap e331: 8 total, 8 up, 8 in 2024-04-26T04:51:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:12 smithi111 ceph-mon[30255]: osdmap e331: 8 total, 8 up, 8 in 2024-04-26T04:51:13.309 INFO:teuthology.orchestra.run.smithi005.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:51:13] "GET /metrics HTTP/1.1" 200 38843 "" "Prometheus/2.43.0" 2024-04-26T04:51:13.742 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap3 2024-04-26T04:51:13.751 INFO:tasks.workunit.client.0.smithi005.stdout:clone image184 -> image186 2024-04-26T04:51:13.901 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 164285573 2024-04-26T04:51:13.915 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:51:13.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[91801]: osdmap e332: 8 total, 8 up, 8 in 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[91801]: pgmap v111: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 9.7 MiB/s rd, 33 MiB/s wr, 507 op/s 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[36961]: osdmap e332: 8 total, 8 up, 8 in 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[36961]: pgmap v111: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 9.7 MiB/s rd, 33 MiB/s wr, 507 op/s 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:13 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:13.983 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:13 smithi111 ceph-mon[30255]: osdmap e332: 8 total, 8 up, 8 in 2024-04-26T04:51:13.984 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:13 smithi111 ceph-mon[30255]: pgmap v111: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 9.7 MiB/s rd, 33 MiB/s wr, 507 op/s 2024-04-26T04:51:13.984 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:13 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:13.984 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:13 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:14.250 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-26T04:51:14.348 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 12900116~2025091 2024-04-26T04:51:14.379 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 4412583~8307949 2024-04-26T04:51:14.405 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 109865092~5822551 2024-04-26T04:51:14.435 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 71743401~4693300 2024-04-26T04:51:14.469 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 132342449~6290439 2024-04-26T04:51:14.500 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 86756363~8236864 2024-04-26T04:51:14.537 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 114311499~4243983 2024-04-26T04:51:14.556 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 82716535~7744621 2024-04-26T04:51:14.589 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user0 2024-04-26T04:51:14.739 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:14 smithi111 ceph-mon[30255]: from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:14.739 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:14 smithi111 ceph-mon[30255]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:14.740 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:14 smithi111 ceph-mon[30255]: osdmap e333: 8 total, 8 up, 8 in 2024-04-26T04:51:14.905 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:14 smithi005 ceph-mon[36961]: from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:14.905 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:14 smithi005 ceph-mon[36961]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:14.905 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:14 smithi005 ceph-mon[36961]: osdmap e333: 8 total, 8 up, 8 in 2024-04-26T04:51:14.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:14 smithi005 ceph-mon[91801]: from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:14.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:14 smithi005 ceph-mon[91801]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:14.905 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:14 smithi005 ceph-mon[91801]: osdmap e333: 8 total, 8 up, 8 in 2024-04-26T04:51:15.157 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:15.723 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 89140862~5760400 2024-04-26T04:51:15.732 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 151917956~4002589 2024-04-26T04:51:15.733 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 91716846~4079848 2024-04-26T04:51:15.788 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 6456160~1068644 2024-04-26T04:51:15.788 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 9578503~263506 2024-04-26T04:51:15.788 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 114570153~1745704 2024-04-26T04:51:15.789 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 161585279~1570148 2024-04-26T04:51:15.789 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 102602647~1985600 2024-04-26T04:51:15.789 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user1 2024-04-26T04:51:15.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:15 smithi005 ceph-mon[91801]: osdmap e334: 8 total, 8 up, 8 in 2024-04-26T04:51:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:15 smithi005 ceph-mon[91801]: pgmap v114: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 9.7 MiB/s rd, 33 MiB/s wr, 507 op/s 2024-04-26T04:51:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:15 smithi005 ceph-mon[36961]: osdmap e334: 8 total, 8 up, 8 in 2024-04-26T04:51:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:15 smithi005 ceph-mon[36961]: pgmap v114: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 9.7 MiB/s rd, 33 MiB/s wr, 507 op/s 2024-04-26T04:51:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:15 smithi111 ceph-mon[30255]: osdmap e334: 8 total, 8 up, 8 in 2024-04-26T04:51:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:15 smithi111 ceph-mon[30255]: pgmap v114: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 9.7 MiB/s rd, 33 MiB/s wr, 507 op/s 2024-04-26T04:51:16.917 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:16 smithi005 ceph-mon[91801]: osdmap e335: 8 total, 8 up, 8 in 2024-04-26T04:51:16.917 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:16.917 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:16 smithi005 ceph-mon[36961]: osdmap e335: 8 total, 8 up, 8 in 2024-04-26T04:51:16.917 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:16 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:16 smithi111 ceph-mon[30255]: osdmap e335: 8 total, 8 up, 8 in 2024-04-26T04:51:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:16 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:17.445 INFO:teuthology.orchestra.run.smithi005.stdout:wait for majority of mons to upgrade 2024-04-26T04:51:17.726 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 289539703 2024-04-26T04:51:17.740 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 149830977~4275315 2024-04-26T04:51:17.748 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 190732356~2047157 2024-04-26T04:51:17.748 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 39203321~1595803 2024-04-26T04:51:17.749 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83413498~6031236 2024-04-26T04:51:17.749 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 89397799~5831295 2024-04-26T04:51:17.811 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 35879576~8068401 2024-04-26T04:51:17.811 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 13032440~7316419 2024-04-26T04:51:17.816 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 73304479~7845380 2024-04-26T04:51:17.824 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user2 2024-04-26T04:51:17.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[91801]: osdmap e336: 8 total, 8 up, 8 in 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[91801]: pgmap v117: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[36961]: osdmap e336: 8 total, 8 up, 8 in 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:17 smithi005 ceph-mon[36961]: pgmap v117: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:17 smithi111 ceph-mon[30255]: osdmap e336: 8 total, 8 up, 8 in 2024-04-26T04:51:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:17 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:17 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:17 smithi111 ceph-mon[30255]: pgmap v117: 137 pgs: 137 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:18.268 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-26T04:51:18.928 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:18.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:18 smithi005 ceph-mon[91801]: osdmap e337: 8 total, 8 up, 8 in 2024-04-26T04:51:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:18 smithi005 ceph-mon[36961]: osdmap e337: 8 total, 8 up, 8 in 2024-04-26T04:51:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:18 smithi111 ceph-mon[30255]: osdmap e337: 8 total, 8 up, 8 in 2024-04-26T04:51:19.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:51:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:51:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:51:19.861 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 302039921 2024-04-26T04:51:19.879 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 152113269~2476555 2024-04-26T04:51:19.890 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 270925352~1708168 2024-04-26T04:51:19.890 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 246523373~1096848 2024-04-26T04:51:19.891 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 2416832~2700570 2024-04-26T04:51:19.892 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 197624155~1513294 2024-04-26T04:51:19.896 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 267677348~1119761 2024-04-26T04:51:19.896 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 282169107~2040028 2024-04-26T04:51:19.899 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 79484591~3186029 2024-04-26T04:51:19.899 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user3 2024-04-26T04:51:20.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:19 smithi005 ceph-mon[91801]: pgmap v119: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 22 MiB/s wr, 636 op/s 2024-04-26T04:51:20.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:19 smithi005 ceph-mon[91801]: osdmap e338: 8 total, 8 up, 8 in 2024-04-26T04:51:20.130 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:19 smithi005 ceph-mon[36961]: pgmap v119: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 22 MiB/s wr, 636 op/s 2024-04-26T04:51:20.130 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:19 smithi005 ceph-mon[36961]: osdmap e338: 8 total, 8 up, 8 in 2024-04-26T04:51:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:19 smithi111 ceph-mon[30255]: pgmap v119: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 22 MiB/s wr, 636 op/s 2024-04-26T04:51:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:19 smithi111 ceph-mon[30255]: osdmap e338: 8 total, 8 up, 8 in 2024-04-26T04:51:21.085 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:20 smithi005 ceph-mon[36961]: osdmap e339: 8 total, 8 up, 8 in 2024-04-26T04:51:21.085 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:20 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:21.085 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:20 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:21.085 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:20 smithi005 ceph-mon[91801]: osdmap e339: 8 total, 8 up, 8 in 2024-04-26T04:51:21.086 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:20 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:21.086 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:20 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:20 smithi111 ceph-mon[30255]: osdmap e339: 8 total, 8 up, 8 in 2024-04-26T04:51:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:20 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:20 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:21.878 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 328150759 2024-04-26T04:51:21.897 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 32815075~8203768 2024-04-26T04:51:21.904 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 34866017~4101884 2024-04-26T04:51:21.957 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 65630151~8203768 2024-04-26T04:51:21.969 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 67681093~4101884 2024-04-26T04:51:22.031 INFO:tasks.workunit.client.0.smithi005.stdout:write: 3 98445227~8203768 2024-04-26T04:51:22.041 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 100496169~4101884 2024-04-26T04:51:22.074 INFO:tasks.workunit.client.0.smithi005.stdout:write: 4 131260303~8203768 2024-04-26T04:51:22.084 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 133311245~4101884 2024-04-26T04:51:22.115 INFO:tasks.workunit.client.0.smithi005.stdout:write: 5 164075379~8203768 2024-04-26T04:51:22.125 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 166126321~4101884 2024-04-26T04:51:22.180 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: pgmap v122: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 22 MiB/s wr, 637 op/s 2024-04-26T04:51:22.180 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: osdmap e340: 8 total, 8 up, 8 in 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3884501412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: pgmap v122: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 22 MiB/s wr, 637 op/s 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: osdmap e340: 8 total, 8 up, 8 in 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:22.181 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:22.182 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:22.182 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:22.182 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:22.182 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:51:22.182 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:51:22.182 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:21 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/3884501412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: pgmap v122: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 23 MiB/s rd, 22 MiB/s wr, 637 op/s 2024-04-26T04:51:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: osdmap e340: 8 total, 8 up, 8 in 2024-04-26T04:51:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:22.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:22.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:51:22.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:51:22.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:21 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3884501412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:23.189 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[91801]: pgmap v124: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[91801]: pgmap v125: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[91801]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[91801]: osdmap e341: 8 total, 8 up, 8 in 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[36961]: pgmap v124: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[36961]: pgmap v125: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[36961]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:51:23.190 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:22 smithi005 ceph-mon[36961]: osdmap e341: 8 total, 8 up, 8 in 2024-04-26T04:51:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:22 smithi111 ceph-mon[30255]: pgmap v124: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:22 smithi111 ceph-mon[30255]: pgmap v125: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-04-26T04:51:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:22 smithi111 ceph-mon[30255]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:51:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:22 smithi111 ceph-mon[30255]: osdmap e341: 8 total, 8 up, 8 in 2024-04-26T04:51:23.343 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:51:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:51:23] "GET /metrics HTTP/1.1" 200 38844 "" "Prometheus/2.43.0" 2024-04-26T04:51:23.549 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:51:24.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: Upgrade: Updating mon.c 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: Deploying daemon mon.c on smithi005 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: pgmap v127: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 70 MiB/s rd, 49 MiB/s wr, 591 op/s 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/913316321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: Upgrade: Updating mon.c 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:51:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:51:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: Deploying daemon mon.c on smithi005 2024-04-26T04:51:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: pgmap v127: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 70 MiB/s rd, 49 MiB/s wr, 591 op/s 2024-04-26T04:51:24.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:24 smithi005 ceph-mon[36961]: from='client.? 172.21.15.5:0/913316321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: Upgrade: Updating mon.c 2024-04-26T04:51:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:51:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:51:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: Deploying daemon mon.c on smithi005 2024-04-26T04:51:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: pgmap v127: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 70 MiB/s rd, 49 MiB/s wr, 591 op/s 2024-04-26T04:51:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:24 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/913316321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:24.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.Stress2 (18943 ms) 2024-04-26T04:51:24.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMigration.StressLive 2024-04-26T04:51:24.856 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 83886080 2024-04-26T04:51:24.866 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 25997901~5674498 2024-04-26T04:51:24.904 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 45060225~5105612 2024-04-26T04:51:24.904 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 80282878~3078582 2024-04-26T04:51:24.930 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 69115257~5707783 2024-04-26T04:51:24.930 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 55496204~1836992 2024-04-26T04:51:24.943 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 77484211~4936793 2024-04-26T04:51:24.945 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 57744996~312568 2024-04-26T04:51:24.949 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 77518209~3205130 2024-04-26T04:51:24.951 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap0 2024-04-26T04:51:25.605 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:25 smithi005 systemd[1]: Stopping Ceph mon.c for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:51:25.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c[36957]: 2024-04-26T04:51:25.603+0000 7fd471614700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c[36957]: 2024-04-26T04:51:25.603+0000 7fd471614700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:25 smithi005 podman[95611]: 2024-04-26 04:51:25.75763123 +0000 UTC m=+0.264332289 container died 78e8c297f4e285ce86f8f3362fff261c6d2584d93a84db6756d4644861c0cb32 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:51:25] ENGINE Bus STOPPING 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:51:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:51:25] ENGINE Bus STOPPED 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:51:25] ENGINE Bus STARTING 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:51:25] ENGINE Serving on http://:::9283 2024-04-26T04:51:25.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:51:25] ENGINE Bus STARTED 2024-04-26T04:51:26.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 podman[95611]: 2024-04-26 04:51:26.237687327 +0000 UTC m=+0.744388389 container cleanup 78e8c297f4e285ce86f8f3362fff261c6d2584d93a84db6756d4644861c0cb32 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:26.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 bash[95611]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c 2024-04-26T04:51:26.826 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 podman[95636]: 2024-04-26 04:51:26.495117725 +0000 UTC m=+0.736287881 container remove 78e8c297f4e285ce86f8f3362fff261c6d2584d93a84db6756d4644861c0cb32 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:26.827 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T04:51:26.827 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 systemd[1]: Stopped Ceph mon.c for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:51:26.827 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.c.service: Consumed 9.274s CPU time. 2024-04-26T04:51:27.099 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:26 smithi005 systemd[1]: Starting Ceph mon.c for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:51:27.099 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 podman[95790]: 2024-04-26 04:51:26.948125464 +0000 UTC m=+0.019454032 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:27.099 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 podman[95790]: 2024-04-26 04:51:27.09903856 +0000 UTC m=+0.170367142 container create 9834a90fbbaaa3e3e9055264d6cad05970748731c137afda54ef380c7231964f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T04:51:27.350 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 podman[95790]: 2024-04-26 04:51:27.318616872 +0000 UTC m=+0.389945422 container init 9834a90fbbaaa3e3e9055264d6cad05970748731c137afda54ef380c7231964f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=) 2024-04-26T04:51:27.350 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 podman[95790]: 2024-04-26 04:51:27.322234011 +0000 UTC m=+0.393562552 container start 9834a90fbbaaa3e3e9055264d6cad05970748731c137afda54ef380c7231964f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:27.350 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:51:27.350 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:51:27.350 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: pidfile_write: ignore empty --pid-file 2024-04-26T04:51:27.350 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: load: jerasure load: lrc 2024-04-26T04:51:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:51:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Git sha 0 2024-04-26T04:51:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: DB SUMMARY 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: DB Session ID: YOMRSQP1Q3U1H32BCROO 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1105 Bytes 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 11600672 ; 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:51:27.734 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.env: 0x55fb63ee4c60 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.info_log: 0x55fb66183aa0 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.statistics: (nil) 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.use_fsync: 0 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:51:27.735 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.db_log_dir: 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.wal_dir: 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:51:27.736 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.write_buffer_manager: 0x55fb661561e0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.unordered_write: 0 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:51:27.737 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.row_cache: None 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.wal_filter: None 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.wal_compression: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:51:27.738 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:51:27.739 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:51:27.740 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_open_files: -1 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:51:27.741 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Compression algorithms supported: 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kZSTD supported: 0 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-26T04:51:27.742 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.merge_operator: 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_filter: None 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fb66183ba0) 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: index_type: 0 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: data_block_index_type: 0 2024-04-26T04:51:27.743 INFO:journalctl@ceph.mon.c.smithi005.stdout: index_shortening: 1 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: checksum: 4 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: no_block_cache: 0 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_cache: 0x55fb66187090 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_cache_options: 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: capacity : 536870912 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: num_shard_bits : 4 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: strict_capacity_limit : 0 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_cache_compressed: (nil) 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: persistent_cache: (nil) 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_size: 4096 2024-04-26T04:51:27.744 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_size_deviation: 10 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_restart_interval: 16 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: index_block_restart_interval: 1 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: metadata_block_size: 4096 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: partition_filters: 0 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: use_delta_encoding: 1 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: filter_policy: bloomfilter 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: whole_key_filtering: 1 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: verify_compression: 0 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: format_version: 5 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: enable_index_compression: 1 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: block_align: 0 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: max_auto_readahead_size: 262144 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: prepopulate_block_cache: 0 2024-04-26T04:51:27.745 INFO:journalctl@ceph.mon.c.smithi005.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression: NoCompression 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.num_levels: 7 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:51:27.746 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:51:27.747 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:51:27.748 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:51:27.749 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:51:27.750 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.ttl: 2592000 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:51:27.751 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13392, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b64eed53-41a9-4522-b56e-02bed66efe41 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107087352617, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107087380752, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7230930, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13397, "largest_seqno": 17659, "table_properties": {"data_size": 7211654, "index_size": 13775, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4485, "raw_key_size": 39363, "raw_average_key_size": 22, "raw_value_size": 7173080, "raw_average_value_size": 4029, "num_data_blocks": 595, "num_entries": 1780, "num_filter_entries": 1780, "num_deletions": 257, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107087, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b64eed53-41a9-4522-b56e-02bed66efe41", "db_session_id": "YOMRSQP1Q3U1H32BCROO", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107087380868, "job": 1, "event": "recovery_finished"} 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:51:27.752 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fb66278000 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: DB pointer 0x55fb66262000 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: ** DB Stats ** 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:51:27.753 INFO:journalctl@ceph.mon.c.smithi005.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: L0 1/0 6.90 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 712.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: L6 1/0 7.71 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: Sum 2/0 14.61 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 712.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 712.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: ** Compaction Stats [default] ** 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 712.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:51:27.754 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Cumulative compaction: 0.01 GB write, 219.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Interval compaction: 0.01 GB write, 219.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Block cache BinnedLRUCache@0x55fb66187090#2 capacity: 512.00 MB usage: 20.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: starting mon.c rank 2 at public addrs [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] at bind addrs [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:51:27.755 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???) e3 preinit fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).mds e9 new map 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).mds e9 print_map 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: e9 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: legacy client fscid: 1 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: Filesystem 'foo' (1) 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: fs_name foo 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: epoch 8 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: created 2024-04-26T04:45:21.622880+0000 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: modified 2024-04-26T04:45:39.686257+0000 2024-04-26T04:51:27.756 INFO:journalctl@ceph.mon.c.smithi005.stdout: tableserver 0 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: root 0 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: session_timeout 60 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: session_autoclose 300 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: max_file_size 1099511627776 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: max_xattr_size 65536 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: required_client_features {} 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: last_failure 0 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: last_failure_osd_epoch 0 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: max_mds 1 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: in 0 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: up {0=24554} 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: failed 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: damaged 2024-04-26T04:51:27.757 INFO:journalctl@ceph.mon.c.smithi005.stdout: stopped 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: data_pools [4] 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: metadata_pool 3 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: inline_data disabled 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: balancer 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: bal_rank_mask -1 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: standby_count_wanted 1 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: [mds.foo.smithi005.zdnlsb{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: Standby daemons: 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout: [mds.foo.smithi111.furhfd{-1:24538} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:51:27.758 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).osd e342 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:51:27.759 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).osd e342 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:51:27.759 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).osd e342 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:51:27.759 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).osd e342 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:51:27.759 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 ceph-mon[95809]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-26T04:51:27.759 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 bash[95790]: 9834a90fbbaaa3e3e9055264d6cad05970748731c137afda54ef380c7231964f 2024-04-26T04:51:27.759 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:27 smithi005 systemd[1]: Started Ceph mon.c for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:51:28.445 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap0 2024-04-26T04:51:28.456 INFO:tasks.workunit.client.0.smithi005.stdout:clone image187 -> image190 2024-04-26T04:51:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 12853980~5474610 2024-04-26T04:51:28.600 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 6987904~6968975 2024-04-26T04:51:28.605 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 8088736~762935 2024-04-26T04:51:28.611 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 19580179~6479962 2024-04-26T04:51:28.644 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 26443207~5095205 2024-04-26T04:51:28.674 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 58616302~6890149 2024-04-26T04:51:28.696 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 71284903~6059216 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1367958392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: pgmap v130: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 40 MiB/s rd, 28 MiB/s wr, 337 op/s 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: mon.c calling monitor election 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: mon.a calling monitor election 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: monmap epoch 3 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: last_changed 2024-04-26T04:40:01.288521+0000 2024-04-26T04:51:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: min_mon_release 17 (quincy) 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: election_strategy: 1 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: osdmap e342: 8 total, 8 up, 8 in 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: mgrmap e37: y(active, since 85s), standbys: x 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: application not enabled on pool 'test-librbd-smithi005-77850-9' 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:51:28.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1367958392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: pgmap v130: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 40 MiB/s rd, 28 MiB/s wr, 337 op/s 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: mon.c calling monitor election 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: mon.a calling monitor election 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: monmap epoch 3 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: last_changed 2024-04-26T04:40:01.288521+0000 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:51:28.732 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: min_mon_release 17 (quincy) 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: election_strategy: 1 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: osdmap e342: 8 total, 8 up, 8 in 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: mgrmap e37: y(active, since 85s), standbys: x 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: application not enabled on pool 'test-librbd-smithi005-77850-9' 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:51:28.733 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:28.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:28.734 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:28 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.740 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 27942044~728123 2024-04-26T04:51:28.742 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap1 2024-04-26T04:51:28.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:51:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:51:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/1367958392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: pgmap v130: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 40 MiB/s rd, 28 MiB/s wr, 337 op/s 2024-04-26T04:51:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: mon.c calling monitor election 2024-04-26T04:51:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: mon.a calling monitor election 2024-04-26T04:51:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: monmap epoch 3 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: last_changed 2024-04-26T04:40:01.288521+0000 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: min_mon_release 17 (quincy) 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: election_strategy: 1 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: osdmap e342: 8 total, 8 up, 8 in 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: mgrmap e37: y(active, since 85s), standbys: x 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:51:28.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:51:28.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: application not enabled on pool 'test-librbd-smithi005-77850-9' 2024-04-26T04:51:28.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:51:28.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:28.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:28.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:28 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:29.032 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T04:51:29.664 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:29 smithi005 ceph-mon[91801]: osdmap e343: 8 total, 8 up, 8 in 2024-04-26T04:51:29.664 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:29 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3833622351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:29.664 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:29 smithi005 ceph-mon[95809]: osdmap e343: 8 total, 8 up, 8 in 2024-04-26T04:51:29.664 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:29 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3833622351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:29 smithi111 ceph-mon[30255]: osdmap e343: 8 total, 8 up, 8 in 2024-04-26T04:51:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:29 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3833622351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:51:30.457 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 108224922 2024-04-26T04:51:30.475 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 45156404~4612400 2024-04-26T04:51:30.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:30 smithi005 ceph-mon[91801]: pgmap v132: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s rd, 26 MiB/s wr, 645 op/s 2024-04-26T04:51:30.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:30 smithi005 ceph-mon[91801]: osdmap e344: 8 total, 8 up, 8 in 2024-04-26T04:51:30.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:30 smithi005 ceph-mon[95809]: pgmap v132: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s rd, 26 MiB/s wr, 645 op/s 2024-04-26T04:51:30.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:30 smithi005 ceph-mon[95809]: osdmap e344: 8 total, 8 up, 8 in 2024-04-26T04:51:30.483 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 32224258~1010010 2024-04-26T04:51:30.483 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 70105613~2361284 2024-04-26T04:51:30.488 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 48999150~1700916 2024-04-26T04:51:30.508 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 90088679~6201879 2024-04-26T04:51:30.511 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 3769214~3150826 2024-04-26T04:51:30.511 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 43651577~807655 2024-04-26T04:51:30.512 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 33788366~695617 2024-04-26T04:51:30.512 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap2 2024-04-26T04:51:30.549 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:51:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:30 smithi111 ceph-mon[30255]: pgmap v132: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s rd, 26 MiB/s wr, 645 op/s 2024-04-26T04:51:30.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:30 smithi111 ceph-mon[30255]: osdmap e344: 8 total, 8 up, 8 in 2024-04-26T04:51:31.363 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:31.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:31 smithi005 ceph-mon[91801]: osdmap e345: 8 total, 8 up, 8 in 2024-04-26T04:51:31.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:31 smithi005 ceph-mon[95809]: osdmap e345: 8 total, 8 up, 8 in 2024-04-26T04:51:31.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:31 smithi111 ceph-mon[30255]: osdmap e345: 8 total, 8 up, 8 in 2024-04-26T04:51:32.466 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap2 2024-04-26T04:51:32.482 INFO:tasks.workunit.client.0.smithi005.stdout:clone image190 -> image192 2024-04-26T04:51:32.616 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 31267242~5886914 2024-04-26T04:51:32.663 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 54896211~5490951 2024-04-26T04:51:32.673 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 83246627~1295031 2024-04-26T04:51:32.719 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 92546290~4834905 2024-04-26T04:51:32.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-mon[91801]: pgmap v135: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s rd, 3.5 MiB/s wr, 459 op/s 2024-04-26T04:51:32.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-mon[91801]: osdmap e346: 8 total, 8 up, 8 in 2024-04-26T04:51:32.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:32.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-mon[95809]: pgmap v135: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s rd, 3.5 MiB/s wr, 459 op/s 2024-04-26T04:51:32.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-mon[95809]: osdmap e346: 8 total, 8 up, 8 in 2024-04-26T04:51:32.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:32.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:32 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:51:32.389+0000 7f15dde08700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-26T04:51:32.739 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 30058428~6115120 2024-04-26T04:51:32.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:32 smithi111 ceph-mon[30255]: pgmap v135: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 157 MiB/s rd, 3.5 MiB/s wr, 459 op/s 2024-04-26T04:51:32.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:32 smithi111 ceph-mon[30255]: osdmap e346: 8 total, 8 up, 8 in 2024-04-26T04:51:32.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:32 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:32.772 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 58361934~4375686 2024-04-26T04:51:32.774 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 8869298~6784144 2024-04-26T04:51:32.828 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 84826803~7346316 2024-04-26T04:51:32.838 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: snap3 2024-04-26T04:51:33.454 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:51:33] "GET /metrics HTTP/1.1" 200 38769 "" "Prometheus/2.43.0" 2024-04-26T04:51:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-mon[91801]: osdmap e347: 8 total, 8 up, 8 in 2024-04-26T04:51:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:33.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-mon[95809]: osdmap e347: 8 total, 8 up, 8 in 2024-04-26T04:51:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:33.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:33 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:33.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:33 smithi111 ceph-mon[30255]: osdmap e347: 8 total, 8 up, 8 in 2024-04-26T04:51:33.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:33 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:33.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:33 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:34.359 INFO:teuthology.orchestra.run.smithi005.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:51:34.359 INFO:teuthology.orchestra.run.smithi005.stdout:alertmanager.a smithi005 *:9093,9094 running (3m) 1s ago 7m 17.0M - 0.25.0 c8568f914cd2 3afeca478730 2024-04-26T04:51:34.359 INFO:teuthology.orchestra.run.smithi005.stdout:grafana.a smithi111 *:3000 error 17s ago 6m - - 2024-04-26T04:51:34.359 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi005.zdnlsb smithi005 running (5m) 1s ago 5m 19.6M - 17.2.7-902-gc61472c8 0de407c1b3af 7ddb93a6b5a1 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi111.furhfd smithi111 running (5m) 17s ago 5m 16.5M - 17.2.7-902-gc61472c8 0de407c1b3af 3e206ca72f2e 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.x smithi111 *:8443,9283,8765 running (116s) 17s ago 11m 449M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0572da61dd7 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.y smithi005 *:8443,9283,8765 running (2m) 1s ago 13m 501M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c895550b1a50 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:mon.a smithi005 running (102s) 1s ago 13m 44.6M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 249028d8f146 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:mon.b smithi111 running (11m) 17s ago 11m 60.1M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 20923c1c0e03 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:mon.c smithi005 running (7s) 1s ago 11m 28.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9834a90fbbaa 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.a smithi005 *:9100 running (2m) 1s ago 6m 9634k - 1.5.0 0da6a335fe13 c74ab514255d 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.b smithi111 *:9100 running (2m) 17s ago 6m 16.6M - 1.5.0 0da6a335fe13 cc0664bb12c9 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:osd.0 smithi005 running (10m) 1s ago 10m 147M 2481M 17.2.7-902-gc61472c8 0de407c1b3af b99cc094c6d7 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:osd.1 smithi005 running (10m) 1s ago 10m 138M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 9e578b598142 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:osd.2 smithi005 running (10m) 1s ago 10m 118M 2481M 17.2.7-902-gc61472c8 0de407c1b3af f25e1cd874ca 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:osd.3 smithi005 running (9m) 1s ago 9m 118M 2481M 17.2.7-902-gc61472c8 0de407c1b3af fd23fb067fc9 2024-04-26T04:51:34.360 INFO:teuthology.orchestra.run.smithi005.stdout:osd.4 smithi111 running (9m) 17s ago 9m 136M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e2994c2a7aa2 2024-04-26T04:51:34.361 INFO:teuthology.orchestra.run.smithi005.stdout:osd.5 smithi111 running (8m) 17s ago 8m 111M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5da203330767 2024-04-26T04:51:34.361 INFO:teuthology.orchestra.run.smithi005.stdout:osd.6 smithi111 running (8m) 17s ago 8m 156M 2480M 17.2.7-902-gc61472c8 0de407c1b3af b7798f2bbff0 2024-04-26T04:51:34.361 INFO:teuthology.orchestra.run.smithi005.stdout:osd.7 smithi111 running (8m) 17s ago 8m 103M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 60f1bafb36a6 2024-04-26T04:51:34.361 INFO:teuthology.orchestra.run.smithi005.stdout:prometheus.a smithi111 *:9095 running (2m) 17s ago 7m 56.2M - 2.43.0 a07b618ecd1d 9e5fc4ef6c62 2024-04-26T04:51:34.484 INFO:tasks.workunit.client.0.smithi005.stdout:snap_protect: snap3 2024-04-26T04:51:34.494 INFO:tasks.workunit.client.0.smithi005.stdout:clone image192 -> image194 2024-04-26T04:51:34.647 INFO:tasks.workunit.client.0.smithi005.stdout:migration_prepare 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[95809]: pgmap v138: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.0 MiB/s rd, 18 MiB/s wr, 342 op/s 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[95809]: osdmap e348: 8 total, 8 up, 8 in 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[91801]: pgmap v138: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.0 MiB/s rd, 18 MiB/s wr, 342 op/s 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[91801]: osdmap e348: 8 total, 8 up, 8 in 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:34.719 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:34 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:34 smithi111 ceph-mon[30255]: pgmap v138: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.0 MiB/s rd, 18 MiB/s wr, 342 op/s 2024-04-26T04:51:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:34 smithi111 ceph-mon[30255]: osdmap e348: 8 total, 8 up, 8 in 2024-04-26T04:51:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:34 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:34.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:34 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:35.058 INFO:tasks.workunit.client.0.smithi005.stdout:migration_execute 2024-04-26T04:51:35.058 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 35470423~2402296 2024-04-26T04:51:35.111 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 61297982~2705017 2024-04-26T04:51:35.139 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 43426397~7410396 2024-04-26T04:51:35.180 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:51:35.191 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 68473484~2330818 2024-04-26T04:51:35.217 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 72497457~5083434 2024-04-26T04:51:35.270 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 100599918~7625004 2024-04-26T04:51:35.302 INFO:tasks.workunit.client.0.smithi005.stdout:write: a 73767350~3422181 2024-04-26T04:51:35.319 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 100357471~7867451 2024-04-26T04:51:35.321 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user0 2024-04-26T04:51:35.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:35 smithi005 ceph-mon[91801]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:35.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:35 smithi005 ceph-mon[91801]: osdmap e349: 8 total, 8 up, 8 in 2024-04-26T04:51:35.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:35 smithi005 ceph-mon[91801]: mgrmap e38: y(active, since 92s), standbys: x 2024-04-26T04:51:35.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:35 smithi005 ceph-mon[95809]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:35.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:35 smithi005 ceph-mon[95809]: osdmap e349: 8 total, 8 up, 8 in 2024-04-26T04:51:35.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:35 smithi005 ceph-mon[95809]: mgrmap e38: y(active, since 92s), standbys: x 2024-04-26T04:51:35.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:35 smithi111 ceph-mon[30255]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:35.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:35 smithi111 ceph-mon[30255]: osdmap e349: 8 total, 8 up, 8 in 2024-04-26T04:51:35.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:35 smithi111 ceph-mon[30255]: mgrmap e38: y(active, since 92s), standbys: x 2024-04-26T04:51:35.956 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:36.489 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 123884472 2024-04-26T04:51:36.503 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 17237113~5911521 2024-04-26T04:51:36.510 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 84969450~6244944 2024-04-26T04:51:36.511 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 115928049~7956423 2024-04-26T04:51:36.516 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 117463111~5171252 2024-04-26T04:51:36.569 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 110016184~2831501 2024-04-26T04:51:36.570 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 6153959~5757506 2024-04-26T04:51:36.570 INFO:tasks.workunit.client.0.smithi005.stdout:write: b 22382355~5447953 2024-04-26T04:51:36.580 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 74651818~119462 2024-04-26T04:51:36.584 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user1 2024-04-26T04:51:36.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:36 smithi005 ceph-mon[91801]: pgmap v141: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.0 MiB/s rd, 18 MiB/s wr, 342 op/s 2024-04-26T04:51:36.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:36 smithi005 ceph-mon[91801]: osdmap e350: 8 total, 8 up, 8 in 2024-04-26T04:51:36.727 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:36 smithi005 ceph-mon[95809]: pgmap v141: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.0 MiB/s rd, 18 MiB/s wr, 342 op/s 2024-04-26T04:51:36.728 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:36 smithi005 ceph-mon[95809]: osdmap e350: 8 total, 8 up, 8 in 2024-04-26T04:51:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:36 smithi111 ceph-mon[30255]: pgmap v141: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.0 MiB/s rd, 18 MiB/s wr, 342 op/s 2024-04-26T04:51:36.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:36 smithi111 ceph-mon[30255]: osdmap e350: 8 total, 8 up, 8 in 2024-04-26T04:51:37.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:37 smithi111 ceph-mon[30255]: osdmap e351: 8 total, 8 up, 8 in 2024-04-26T04:51:37.899 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:37 smithi005 ceph-mon[91801]: osdmap e351: 8 total, 8 up, 8 in 2024-04-26T04:51:37.900 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:37 smithi005 ceph-mon[95809]: osdmap e351: 8 total, 8 up, 8 in 2024-04-26T04:51:38.507 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 184198277 2024-04-26T04:51:38.519 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 113827178~5620205 2024-04-26T04:51:38.529 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 98451809~7359303 2024-04-26T04:51:38.529 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 134882512~5344016 2024-04-26T04:51:38.534 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 103147480~7388993 2024-04-26T04:51:38.569 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 49161975~3330401 2024-04-26T04:51:38.569 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 134111817~902114 2024-04-26T04:51:38.570 INFO:tasks.workunit.client.0.smithi005.stdout:write: c 101194918~7641710 2024-04-26T04:51:38.583 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 176623580~7574697 2024-04-26T04:51:38.583 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user2 2024-04-26T04:51:38.679 INFO:teuthology.orchestra.run.smithi005.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:38.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:38 smithi005 ceph-mon[95809]: pgmap v144: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:38.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:38 smithi005 ceph-mon[95809]: osdmap e352: 8 total, 8 up, 8 in 2024-04-26T04:51:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:38 smithi005 ceph-mon[91801]: pgmap v144: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:38.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:38 smithi005 ceph-mon[91801]: osdmap e352: 8 total, 8 up, 8 in 2024-04-26T04:51:38.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:38 smithi111 ceph-mon[30255]: pgmap v144: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:38.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:38 smithi111 ceph-mon[30255]: osdmap e352: 8 total, 8 up, 8 in 2024-04-26T04:51:38.768 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:51:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:51:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:51:39.394 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[91801]: osdmap e353: 8 total, 8 up, 8 in 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[91801]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[91801]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[95809]: osdmap e353: 8 total, 8 up, 8 in 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[95809]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[95809]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:39.682 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:39 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:39.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:39 smithi111 ceph-mon[30255]: osdmap e353: 8 total, 8 up, 8 in 2024-04-26T04:51:39.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:39 smithi111 ceph-mon[30255]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:39.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:39 smithi111 ceph-mon[30255]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:39.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:39 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.044 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:40.513 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 8081167~7142512 2024-04-26T04:51:40.528 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 116264362~7234314 2024-04-26T04:51:40.528 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 71487926~2172961 2024-04-26T04:51:40.529 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 75154450~2474762 2024-04-26T04:51:40.529 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 3299254~34011 2024-04-26T04:51:40.529 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 134016101~7111751 2024-04-26T04:51:40.557 INFO:tasks.workunit.client.0.smithi005.stdout:write: d 149912635~2232720 2024-04-26T04:51:40.559 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 16098800~5418579 2024-04-26T04:51:40.568 INFO:tasks.workunit.client.0.smithi005.stdout:snap_create: user3 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: pgmap v147: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 51 MiB/s wr, 1.03k op/s 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: osdmap e354: 8 total, 8 up, 8 in 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: pgmap v149: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 41 MiB/s rd, 57 MiB/s wr, 1.15k op/s 2024-04-26T04:51:40.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: pgmap v150: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 52 MiB/s rd, 74 MiB/s wr, 1.49k op/s 2024-04-26T04:51:40.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:40.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:40 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: pgmap v147: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 51 MiB/s wr, 1.03k op/s 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: osdmap e354: 8 total, 8 up, 8 in 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: pgmap v149: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 41 MiB/s rd, 57 MiB/s wr, 1.15k op/s 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: pgmap v150: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 52 MiB/s rd, 74 MiB/s wr, 1.49k op/s 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: pgmap v147: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 51 MiB/s wr, 1.03k op/s 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: osdmap e354: 8 total, 8 up, 8 in 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: pgmap v149: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 41 MiB/s rd, 57 MiB/s wr, 1.15k op/s 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: pgmap v150: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 52 MiB/s rd, 74 MiB/s wr, 1.49k op/s 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:40.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:40.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:40 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:41.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:41 smithi005 ceph-mon[91801]: pgmap v151: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 54 MiB/s rd, 76 MiB/s wr, 1.52k op/s 2024-04-26T04:51:41.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:41 smithi005 ceph-mon[91801]: pgmap v152: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:41.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:41 smithi005 ceph-mon[91801]: osdmap e355: 8 total, 8 up, 8 in 2024-04-26T04:51:41.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:41 smithi005 ceph-mon[95809]: pgmap v151: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 54 MiB/s rd, 76 MiB/s wr, 1.52k op/s 2024-04-26T04:51:41.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:41 smithi005 ceph-mon[95809]: pgmap v152: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:41.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:41 smithi005 ceph-mon[95809]: osdmap e355: 8 total, 8 up, 8 in 2024-04-26T04:51:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:41 smithi111 ceph-mon[30255]: pgmap v151: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 54 MiB/s rd, 76 MiB/s wr, 1.52k op/s 2024-04-26T04:51:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:41 smithi111 ceph-mon[30255]: pgmap v152: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:41 smithi111 ceph-mon[30255]: osdmap e355: 8 total, 8 up, 8 in 2024-04-26T04:51:42.529 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 124698240 2024-04-26T04:51:42.564 INFO:tasks.workunit.client.0.smithi005.stdout:write: 1 12469824~3117456 2024-04-26T04:51:42.565 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 13249188~1558728 2024-04-26T04:51:42.583 INFO:tasks.workunit.client.0.smithi005.stdout:write: 2 24939648~3117456 2024-04-26T04:51:42.584 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 25719012~1558728 2024-04-26T04:51:42.599 INFO:tasks.workunit.client.0.smithi005.stdout:write: 3 37409472~3117456 2024-04-26T04:51:42.599 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 38188836~1558728 2024-04-26T04:51:42.618 INFO:tasks.workunit.client.0.smithi005.stdout:write: 4 49879296~3117456 2024-04-26T04:51:42.619 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 50658660~1558728 2024-04-26T04:51:42.634 INFO:tasks.workunit.client.0.smithi005.stdout:write: 5 62349120~3117456 2024-04-26T04:51:42.635 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 63128484~1558728 2024-04-26T04:51:42.745 INFO:tasks.workunit.client.0.smithi005.stdout:migration_commit 2024-04-26T04:51:42.767 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:42 smithi111 ceph-mon[30255]: osdmap e356: 8 total, 8 up, 8 in 2024-04-26T04:51:42.943 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:42 smithi005 ceph-mon[91801]: osdmap e356: 8 total, 8 up, 8 in 2024-04-26T04:51:42.943 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:42 smithi005 ceph-mon[95809]: osdmap e356: 8 total, 8 up, 8 in 2024-04-26T04:51:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:51:43] "GET /metrics HTTP/1.1" 200 38769 "" "Prometheus/2.43.0" 2024-04-26T04:51:43.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:43 smithi111 ceph-mon[30255]: pgmap v155: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:43.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:43 smithi111 ceph-mon[30255]: osdmap e357: 8 total, 8 up, 8 in 2024-04-26T04:51:43.890 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMigration.StressLive (19066 ms) 2024-04-26T04:51:43.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:43 smithi005 ceph-mon[91801]: pgmap v155: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:43.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:43 smithi005 ceph-mon[91801]: osdmap e357: 8 total, 8 up, 8 in 2024-04-26T04:51:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:43 smithi005 ceph-mon[95809]: pgmap v155: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:43 smithi005 ceph-mon[95809]: osdmap e357: 8 total, 8 up, 8 in 2024-04-26T04:51:45.528 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 57 tests from TestMigration (261794 ms total) 2024-04-26T04:51:45.528 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:51:45.528 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 6 tests from TestGroup 2024-04-26T04:51:45.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:45 smithi005 ceph-mon[95809]: pgmap v157: 137 pgs: 137 active+clean; 2.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 18 MiB/s wr, 213 op/s 2024-04-26T04:51:45.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:45 smithi005 ceph-mon[95809]: osdmap e358: 8 total, 8 up, 8 in 2024-04-26T04:51:45.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:45 smithi005 ceph-mon[91801]: pgmap v157: 137 pgs: 137 active+clean; 2.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 18 MiB/s wr, 213 op/s 2024-04-26T04:51:45.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:45 smithi005 ceph-mon[91801]: osdmap e358: 8 total, 8 up, 8 in 2024-04-26T04:51:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:45 smithi111 ceph-mon[30255]: pgmap v157: 137 pgs: 137 active+clean; 2.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 2.1 MiB/s rd, 18 MiB/s wr, 213 op/s 2024-04-26T04:51:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:45 smithi111 ceph-mon[30255]: osdmap e358: 8 total, 8 up, 8 in 2024-04-26T04:51:46.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:46 smithi005 ceph-mon[91801]: osdmap e359: 8 total, 8 up, 8 in 2024-04-26T04:51:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:46 smithi005 ceph-mon[95809]: osdmap e359: 8 total, 8 up, 8 in 2024-04-26T04:51:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:46 smithi111 ceph-mon[30255]: osdmap e359: 8 total, 8 up, 8 in 2024-04-26T04:51:47.535 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestGroup.group_create 2024-04-26T04:51:47.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[91801]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[91801]: osdmap e360: 8 total, 8 up, 8 in 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/390418765' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[95809]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[95809]: osdmap e360: 8 total, 8 up, 8 in 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/390418765' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:47 smithi111 ceph-mon[30255]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-04-26T04:51:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:47 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:51:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:47 smithi111 ceph-mon[30255]: osdmap e360: 8 total, 8 up, 8 in 2024-04-26T04:51:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:47 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/390418765' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:51:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:47 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:51:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:47 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:48.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:48 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/390418765' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:51:48.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:48 smithi005 ceph-mon[91801]: osdmap e361: 8 total, 8 up, 8 in 2024-04-26T04:51:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:48 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/390418765' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:51:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:48 smithi005 ceph-mon[95809]: osdmap e361: 8 total, 8 up, 8 in 2024-04-26T04:51:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:48 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/390418765' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:51:49.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:48 smithi111 ceph-mon[30255]: osdmap e361: 8 total, 8 up, 8 in 2024-04-26T04:51:49.019 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:51:48 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:51:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:51:49.572 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestGroup.group_create (2037 ms) 2024-04-26T04:51:49.572 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestGroup.group_createPP 2024-04-26T04:51:49.591 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestGroup.group_createPP (19 ms) 2024-04-26T04:51:49.591 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestGroup.add_image 2024-04-26T04:51:49.632 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestGroup.add_image (40 ms) 2024-04-26T04:51:49.632 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestGroup.add_imagePP 2024-04-26T04:51:49.668 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestGroup.add_imagePP (37 ms) 2024-04-26T04:51:49.668 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestGroup.add_snapshot 2024-04-26T04:51:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:49 smithi005 ceph-mon[91801]: pgmap v163: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:49 smithi005 ceph-mon[91801]: osdmap e362: 8 total, 8 up, 8 in 2024-04-26T04:51:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:49 smithi005 ceph-mon[95809]: pgmap v163: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:49 smithi005 ceph-mon[95809]: osdmap e362: 8 total, 8 up, 8 in 2024-04-26T04:51:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:49 smithi111 ceph-mon[30255]: pgmap v163: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:49 smithi111 ceph-mon[30255]: osdmap e362: 8 total, 8 up, 8 in 2024-04-26T04:51:50.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:50 smithi005 ceph-mon[91801]: osdmap e363: 8 total, 8 up, 8 in 2024-04-26T04:51:50.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:50 smithi005 ceph-mon[95809]: osdmap e363: 8 total, 8 up, 8 in 2024-04-26T04:51:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:50 smithi111 ceph-mon[30255]: osdmap e363: 8 total, 8 up, 8 in 2024-04-26T04:51:51.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:51 smithi005 ceph-mon[95809]: pgmap v166: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:51.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:51 smithi005 ceph-mon[95809]: osdmap e364: 8 total, 8 up, 8 in 2024-04-26T04:51:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:51 smithi005 ceph-mon[91801]: pgmap v166: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:51 smithi005 ceph-mon[91801]: osdmap e364: 8 total, 8 up, 8 in 2024-04-26T04:51:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:51 smithi111 ceph-mon[30255]: pgmap v166: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:51 smithi111 ceph-mon[30255]: osdmap e364: 8 total, 8 up, 8 in 2024-04-26T04:51:52.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:52 smithi005 ceph-mon[95809]: osdmap e365: 8 total, 8 up, 8 in 2024-04-26T04:51:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:52 smithi005 ceph-mon[91801]: osdmap e365: 8 total, 8 up, 8 in 2024-04-26T04:51:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:52 smithi111 ceph-mon[30255]: osdmap e365: 8 total, 8 up, 8 in 2024-04-26T04:51:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:51:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:51:53] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.43.0" 2024-04-26T04:51:53.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:53 smithi005 ceph-mon[91801]: pgmap v169: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:53.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:53 smithi005 ceph-mon[91801]: osdmap e366: 8 total, 8 up, 8 in 2024-04-26T04:51:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:53 smithi005 ceph-mon[95809]: pgmap v169: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:53 smithi005 ceph-mon[95809]: osdmap e366: 8 total, 8 up, 8 in 2024-04-26T04:51:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:53 smithi111 ceph-mon[30255]: pgmap v169: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:51:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:53 smithi111 ceph-mon[30255]: osdmap e366: 8 total, 8 up, 8 in 2024-04-26T04:51:54.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:54 smithi005 ceph-mon[95809]: osdmap e367: 8 total, 8 up, 8 in 2024-04-26T04:51:54.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:54 smithi005 ceph-mon[91801]: osdmap e367: 8 total, 8 up, 8 in 2024-04-26T04:51:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:54 smithi111 ceph-mon[30255]: osdmap e367: 8 total, 8 up, 8 in 2024-04-26T04:51:55.612 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestGroup.add_snapshot (5944 ms) 2024-04-26T04:51:55.612 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-04-26T04:51:55.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:55 smithi005 ceph-mon[91801]: pgmap v172: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-04-26T04:51:55.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:55 smithi005 ceph-mon[91801]: osdmap e368: 8 total, 8 up, 8 in 2024-04-26T04:51:55.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:55 smithi005 ceph-mon[95809]: pgmap v172: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-04-26T04:51:55.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:55 smithi005 ceph-mon[95809]: osdmap e368: 8 total, 8 up, 8 in 2024-04-26T04:51:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:55 smithi111 ceph-mon[30255]: pgmap v172: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-04-26T04:51:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:55 smithi111 ceph-mon[30255]: osdmap e368: 8 total, 8 up, 8 in 2024-04-26T04:51:56.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:56 smithi005 ceph-mon[91801]: osdmap e369: 8 total, 8 up, 8 in 2024-04-26T04:51:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:56 smithi005 ceph-mon[95809]: osdmap e369: 8 total, 8 up, 8 in 2024-04-26T04:51:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:56 smithi111 ceph-mon[30255]: osdmap e369: 8 total, 8 up, 8 in 2024-04-26T04:51:57.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:57 smithi005 ceph-mon[91801]: pgmap v175: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-04-26T04:51:57.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:57 smithi005 ceph-mon[91801]: osdmap e370: 8 total, 8 up, 8 in 2024-04-26T04:51:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:57 smithi005 ceph-mon[95809]: pgmap v175: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-04-26T04:51:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:57 smithi005 ceph-mon[95809]: osdmap e370: 8 total, 8 up, 8 in 2024-04-26T04:51:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:57 smithi111 ceph-mon[30255]: pgmap v175: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-04-26T04:51:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:57 smithi111 ceph-mon[30255]: osdmap e370: 8 total, 8 up, 8 in 2024-04-26T04:51:58.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:58 smithi005 ceph-mon[91801]: osdmap e371: 8 total, 8 up, 8 in 2024-04-26T04:51:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:58 smithi005 ceph-mon[91801]: osdmap e372: 8 total, 8 up, 8 in 2024-04-26T04:51:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:58 smithi005 ceph-mon[95809]: osdmap e371: 8 total, 8 up, 8 in 2024-04-26T04:51:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:58 smithi005 ceph-mon[95809]: osdmap e372: 8 total, 8 up, 8 in 2024-04-26T04:51:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:58 smithi111 ceph-mon[30255]: osdmap e371: 8 total, 8 up, 8 in 2024-04-26T04:51:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:58 smithi111 ceph-mon[30255]: osdmap e372: 8 total, 8 up, 8 in 2024-04-26T04:51:59.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:51:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:51:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:51:59.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:59 smithi005 ceph-mon[91801]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 13 KiB/s wr, 162 op/s 2024-04-26T04:51:59.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:51:59 smithi005 ceph-mon[91801]: osdmap e373: 8 total, 8 up, 8 in 2024-04-26T04:51:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:59 smithi005 ceph-mon[95809]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 13 KiB/s wr, 162 op/s 2024-04-26T04:51:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:51:59 smithi005 ceph-mon[95809]: osdmap e373: 8 total, 8 up, 8 in 2024-04-26T04:52:00.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:59 smithi111 ceph-mon[30255]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 13 KiB/s wr, 162 op/s 2024-04-26T04:52:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:51:59 smithi111 ceph-mon[30255]: osdmap e373: 8 total, 8 up, 8 in 2024-04-26T04:52:00.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:00 smithi111 ceph-mon[30255]: osdmap e374: 8 total, 8 up, 8 in 2024-04-26T04:52:00.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:00 smithi005 ceph-mon[91801]: osdmap e374: 8 total, 8 up, 8 in 2024-04-26T04:52:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:00 smithi005 ceph-mon[95809]: osdmap e374: 8 total, 8 up, 8 in 2024-04-26T04:52:01.667 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestGroup.add_snapshotPP (6055 ms) 2024-04-26T04:52:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[91801]: pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 13 KiB/s wr, 163 op/s 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[91801]: osdmap e375: 8 total, 8 up, 8 in 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[95809]: pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 13 KiB/s wr, 163 op/s 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[95809]: osdmap e375: 8 total, 8 up, 8 in 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:01 smithi111 ceph-mon[30255]: pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 13 KiB/s wr, 163 op/s 2024-04-26T04:52:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:01 smithi111 ceph-mon[30255]: osdmap e375: 8 total, 8 up, 8 in 2024-04-26T04:52:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:01 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:01 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:02.650 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 6 tests from TestGroup (14132 ms total) 2024-04-26T04:52:02.651 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:52:02.651 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 31 tests from TestDeepCopy 2024-04-26T04:52:02.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:02 smithi005 ceph-mon[91801]: osdmap e376: 8 total, 8 up, 8 in 2024-04-26T04:52:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:02 smithi005 ceph-mon[95809]: osdmap e376: 8 total, 8 up, 8 in 2024-04-26T04:52:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:02 smithi111 ceph-mon[30255]: osdmap e376: 8 total, 8 up, 8 in 2024-04-26T04:52:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:52:03] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.43.0" 2024-04-26T04:52:03.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:03 smithi005 ceph-mon[91801]: pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:03 smithi005 ceph-mon[91801]: osdmap e377: 8 total, 8 up, 8 in 2024-04-26T04:52:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:03 smithi005 ceph-mon[95809]: pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:03 smithi005 ceph-mon[95809]: osdmap e377: 8 total, 8 up, 8 in 2024-04-26T04:52:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:03 smithi111 ceph-mon[30255]: pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:03 smithi111 ceph-mon[30255]: osdmap e377: 8 total, 8 up, 8 in 2024-04-26T04:52:04.653 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Empty 2024-04-26T04:52:04.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[95809]: osdmap e378: 8 total, 8 up, 8 in 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3418490466' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3418490466' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[95809]: osdmap e379: 8 total, 8 up, 8 in 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[91801]: osdmap e378: 8 total, 8 up, 8 in 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3418490466' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3418490466' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:52:04.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:04 smithi005 ceph-mon[91801]: osdmap e379: 8 total, 8 up, 8 in 2024-04-26T04:52:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:04 smithi111 ceph-mon[30255]: osdmap e378: 8 total, 8 up, 8 in 2024-04-26T04:52:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:04 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3418490466' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:52:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:04 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/3418490466' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:52:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:04 smithi111 ceph-mon[30255]: osdmap e379: 8 total, 8 up, 8 in 2024-04-26T04:52:05.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:05 smithi005 ceph-mon[91801]: pgmap v188: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:05.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:05 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:52:05.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:05 smithi005 ceph-mon[91801]: osdmap e380: 8 total, 8 up, 8 in 2024-04-26T04:52:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:05 smithi005 ceph-mon[95809]: pgmap v188: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:05 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:52:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:05 smithi005 ceph-mon[95809]: osdmap e380: 8 total, 8 up, 8 in 2024-04-26T04:52:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:05 smithi111 ceph-mon[30255]: pgmap v188: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:05 smithi111 ceph-mon[30255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:52:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:05 smithi111 ceph-mon[30255]: osdmap e380: 8 total, 8 up, 8 in 2024-04-26T04:52:06.800 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Empty (2147 ms) 2024-04-26T04:52:06.800 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-04-26T04:52:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:06 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:52:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:06 smithi005 ceph-mon[91801]: osdmap e381: 8 total, 8 up, 8 in 2024-04-26T04:52:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:06 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:52:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:06 smithi005 ceph-mon[95809]: osdmap e381: 8 total, 8 up, 8 in 2024-04-26T04:52:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:06 smithi111 ceph-mon[30255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:52:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:06 smithi111 ceph-mon[30255]: osdmap e381: 8 total, 8 up, 8 in 2024-04-26T04:52:07.049 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.NoSnaps (249 ms) 2024-04-26T04:52:07.049 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Snaps 2024-04-26T04:52:07.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:07 smithi005 ceph-mon[91801]: pgmap v191: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:07 smithi005 ceph-mon[95809]: pgmap v191: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:07 smithi111 ceph-mon[30255]: pgmap v191: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:08 smithi111 ceph-mon[30255]: osdmap e382: 8 total, 8 up, 8 in 2024-04-26T04:52:09.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:52:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:52:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:52:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:08 smithi005 ceph-mon[91801]: osdmap e382: 8 total, 8 up, 8 in 2024-04-26T04:52:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:08 smithi005 ceph-mon[95809]: osdmap e382: 8 total, 8 up, 8 in 2024-04-26T04:52:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:09 smithi111 ceph-mon[30255]: pgmap v194: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 127 op/s 2024-04-26T04:52:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:09 smithi111 ceph-mon[30255]: osdmap e383: 8 total, 8 up, 8 in 2024-04-26T04:52:10.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:09 smithi005 ceph-mon[95809]: pgmap v194: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 127 op/s 2024-04-26T04:52:10.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:09 smithi005 ceph-mon[95809]: osdmap e383: 8 total, 8 up, 8 in 2024-04-26T04:52:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:09 smithi005 ceph-mon[91801]: pgmap v194: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 127 op/s 2024-04-26T04:52:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:09 smithi005 ceph-mon[91801]: osdmap e383: 8 total, 8 up, 8 in 2024-04-26T04:52:10.974 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Snaps (3925 ms) 2024-04-26T04:52:10.975 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-04-26T04:52:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:10 smithi111 ceph-mon[30255]: osdmap e384: 8 total, 8 up, 8 in 2024-04-26T04:52:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:10 smithi005 ceph-mon[91801]: osdmap e384: 8 total, 8 up, 8 in 2024-04-26T04:52:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:10 smithi005 ceph-mon[95809]: osdmap e384: 8 total, 8 up, 8 in 2024-04-26T04:52:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:11 smithi111 ceph-mon[30255]: pgmap v197: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 127 op/s 2024-04-26T04:52:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:11 smithi111 ceph-mon[30255]: osdmap e385: 8 total, 8 up, 8 in 2024-04-26T04:52:12.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:11 smithi005 ceph-mon[91801]: pgmap v197: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 127 op/s 2024-04-26T04:52:12.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:11 smithi005 ceph-mon[91801]: osdmap e385: 8 total, 8 up, 8 in 2024-04-26T04:52:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:11 smithi005 ceph-mon[95809]: pgmap v197: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 10 MiB/s wr, 127 op/s 2024-04-26T04:52:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:11 smithi005 ceph-mon[95809]: osdmap e385: 8 total, 8 up, 8 in 2024-04-26T04:52:12.733 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.SnapDiscard (1760 ms) 2024-04-26T04:52:12.734 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-04-26T04:52:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:12 smithi111 ceph-mon[30255]: osdmap e386: 8 total, 8 up, 8 in 2024-04-26T04:52:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:12 smithi111 ceph-mon[30255]: osdmap e387: 8 total, 8 up, 8 in 2024-04-26T04:52:13.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:12 smithi005 ceph-mon[91801]: osdmap e386: 8 total, 8 up, 8 in 2024-04-26T04:52:13.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:12 smithi005 ceph-mon[91801]: osdmap e387: 8 total, 8 up, 8 in 2024-04-26T04:52:13.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:12 smithi005 ceph-mon[95809]: osdmap e386: 8 total, 8 up, 8 in 2024-04-26T04:52:13.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:12 smithi005 ceph-mon[95809]: osdmap e387: 8 total, 8 up, 8 in 2024-04-26T04:52:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:52:13] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.43.0" 2024-04-26T04:52:13.964 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneDiscard (1230 ms) 2024-04-26T04:52:13.964 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-04-26T04:52:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:13 smithi111 ceph-mon[30255]: pgmap v200: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:14.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:13 smithi005 ceph-mon[91801]: pgmap v200: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:13 smithi005 ceph-mon[95809]: pgmap v200: 105 pgs: 105 active+clean; 41 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:14.939 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneShrink (975 ms) 2024-04-26T04:52:14.939 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-04-26T04:52:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:14 smithi111 ceph-mon[30255]: osdmap e388: 8 total, 8 up, 8 in 2024-04-26T04:52:15.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:14 smithi005 ceph-mon[91801]: osdmap e388: 8 total, 8 up, 8 in 2024-04-26T04:52:15.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:14 smithi005 ceph-mon[95809]: osdmap e388: 8 total, 8 up, 8 in 2024-04-26T04:52:15.998 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneExpand (1059 ms) 2024-04-26T04:52:15.998 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-04-26T04:52:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:15 smithi111 ceph-mon[30255]: pgmap v203: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.0 MiB/s wr, 147 op/s 2024-04-26T04:52:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:15 smithi111 ceph-mon[30255]: osdmap e389: 8 total, 8 up, 8 in 2024-04-26T04:52:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:15 smithi005 ceph-mon[91801]: pgmap v203: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.0 MiB/s wr, 147 op/s 2024-04-26T04:52:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:15 smithi005 ceph-mon[91801]: osdmap e389: 8 total, 8 up, 8 in 2024-04-26T04:52:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:15 smithi005 ceph-mon[95809]: pgmap v203: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.0 MiB/s wr, 147 op/s 2024-04-26T04:52:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:15 smithi005 ceph-mon[95809]: osdmap e389: 8 total, 8 up, 8 in 2024-04-26T04:52:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:16 smithi005 ceph-mon[91801]: osdmap e390: 8 total, 8 up, 8 in 2024-04-26T04:52:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:16 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:16 smithi005 ceph-mon[95809]: osdmap e390: 8 total, 8 up, 8 in 2024-04-26T04:52:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:16 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:16 smithi111 ceph-mon[30255]: osdmap e390: 8 total, 8 up, 8 in 2024-04-26T04:52:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:16 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:16 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:18.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:17 smithi005 ceph-mon[91801]: pgmap v206: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.0 MiB/s wr, 147 op/s 2024-04-26T04:52:18.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:17 smithi005 ceph-mon[91801]: osdmap e391: 8 total, 8 up, 8 in 2024-04-26T04:52:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:17 smithi005 ceph-mon[95809]: pgmap v206: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.0 MiB/s wr, 147 op/s 2024-04-26T04:52:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:17 smithi005 ceph-mon[95809]: osdmap e391: 8 total, 8 up, 8 in 2024-04-26T04:52:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:17 smithi111 ceph-mon[30255]: pgmap v206: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.0 MiB/s wr, 147 op/s 2024-04-26T04:52:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:17 smithi111 ceph-mon[30255]: osdmap e391: 8 total, 8 up, 8 in 2024-04-26T04:52:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:18 smithi111 ceph-mon[30255]: osdmap e392: 8 total, 8 up, 8 in 2024-04-26T04:52:19.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:52:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:52:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:52:19.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:18 smithi005 ceph-mon[91801]: osdmap e392: 8 total, 8 up, 8 in 2024-04-26T04:52:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:18 smithi005 ceph-mon[95809]: osdmap e392: 8 total, 8 up, 8 in 2024-04-26T04:52:20.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:19 smithi005 ceph-mon[91801]: pgmap v209: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-04-26T04:52:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:19 smithi005 ceph-mon[91801]: osdmap e393: 8 total, 8 up, 8 in 2024-04-26T04:52:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:19 smithi005 ceph-mon[95809]: pgmap v209: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-04-26T04:52:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:19 smithi005 ceph-mon[95809]: osdmap e393: 8 total, 8 up, 8 in 2024-04-26T04:52:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:19 smithi111 ceph-mon[30255]: pgmap v209: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-04-26T04:52:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:19 smithi111 ceph-mon[30255]: osdmap e393: 8 total, 8 up, 8 in 2024-04-26T04:52:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:20 smithi005 ceph-mon[91801]: osdmap e394: 8 total, 8 up, 8 in 2024-04-26T04:52:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:20 smithi005 ceph-mon[95809]: osdmap e394: 8 total, 8 up, 8 in 2024-04-26T04:52:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:20 smithi111 ceph-mon[30255]: osdmap e394: 8 total, 8 up, 8 in 2024-04-26T04:52:22.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:21 smithi005 ceph-mon[91801]: pgmap v212: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-04-26T04:52:22.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:21 smithi005 ceph-mon[91801]: osdmap e395: 8 total, 8 up, 8 in 2024-04-26T04:52:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:21 smithi005 ceph-mon[95809]: pgmap v212: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-04-26T04:52:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:21 smithi005 ceph-mon[95809]: osdmap e395: 8 total, 8 up, 8 in 2024-04-26T04:52:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:21 smithi111 ceph-mon[30255]: pgmap v212: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-04-26T04:52:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:21 smithi111 ceph-mon[30255]: osdmap e395: 8 total, 8 up, 8 in 2024-04-26T04:52:22.927 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneHideParent (6929 ms) 2024-04-26T04:52:22.927 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Clone 2024-04-26T04:52:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:22 smithi005 ceph-mon[91801]: osdmap e396: 8 total, 8 up, 8 in 2024-04-26T04:52:23.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:22 smithi005 ceph-mon[95809]: osdmap e396: 8 total, 8 up, 8 in 2024-04-26T04:52:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:22 smithi111 ceph-mon[30255]: osdmap e396: 8 total, 8 up, 8 in 2024-04-26T04:52:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:52:23] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.43.0" 2024-04-26T04:52:24.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:23 smithi005 ceph-mon[91801]: pgmap v215: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:24.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:23 smithi005 ceph-mon[91801]: osdmap e397: 8 total, 8 up, 8 in 2024-04-26T04:52:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:23 smithi005 ceph-mon[95809]: pgmap v215: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:23 smithi005 ceph-mon[95809]: osdmap e397: 8 total, 8 up, 8 in 2024-04-26T04:52:24.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:23 smithi111 ceph-mon[30255]: pgmap v215: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:52:24.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:23 smithi111 ceph-mon[30255]: osdmap e397: 8 total, 8 up, 8 in 2024-04-26T04:52:25.168 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Clone (2240 ms) 2024-04-26T04:52:25.168 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-04-26T04:52:25.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:24 smithi005 ceph-mon[91801]: osdmap e398: 8 total, 8 up, 8 in 2024-04-26T04:52:25.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:24 smithi005 ceph-mon[95809]: osdmap e398: 8 total, 8 up, 8 in 2024-04-26T04:52:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:24 smithi111 ceph-mon[30255]: osdmap e398: 8 total, 8 up, 8 in 2024-04-26T04:52:26.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:25 smithi005 ceph-mon[91801]: pgmap v218: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-04-26T04:52:26.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:25 smithi005 ceph-mon[91801]: osdmap e399: 8 total, 8 up, 8 in 2024-04-26T04:52:26.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:25 smithi005 ceph-mon[95809]: pgmap v218: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-04-26T04:52:26.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:25 smithi005 ceph-mon[95809]: osdmap e399: 8 total, 8 up, 8 in 2024-04-26T04:52:26.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:25 smithi111 ceph-mon[30255]: pgmap v218: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-04-26T04:52:26.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:25 smithi111 ceph-mon[30255]: osdmap e399: 8 total, 8 up, 8 in 2024-04-26T04:52:27.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:26 smithi005 ceph-mon[91801]: osdmap e400: 8 total, 8 up, 8 in 2024-04-26T04:52:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:26 smithi005 ceph-mon[95809]: osdmap e400: 8 total, 8 up, 8 in 2024-04-26T04:52:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:26 smithi111 ceph-mon[30255]: osdmap e400: 8 total, 8 up, 8 in 2024-04-26T04:52:27.292 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneFlatten (2125 ms) 2024-04-26T04:52:27.292 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Stress 2024-04-26T04:52:27.314 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 26083048~3541976 2024-04-26T04:52:27.329 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 58979488~2119210 2024-04-26T04:52:27.330 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 55425525~3090186 2024-04-26T04:52:27.342 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 68802731~2733994 2024-04-26T04:52:27.343 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 55311600~5127888 2024-04-26T04:52:27.357 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 4926262~417010 2024-04-26T04:52:27.358 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 57001661~7212672 2024-04-26T04:52:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 20151582~1956233 2024-04-26T04:52:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap0 2024-04-26T04:52:27.851 INFO:tasks.workunit.client.0.smithi005.stdout:clone image240 -> image241 2024-04-26T04:52:27.895 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 65656147~4880076 2024-04-26T04:52:27.916 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 73872240~8035232 2024-04-26T04:52:27.920 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 55191496~5308472 2024-04-26T04:52:27.955 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 27687670~1160182 2024-04-26T04:52:27.966 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 44313665~937566 2024-04-26T04:52:27.970 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 10901938~7309918 2024-04-26T04:52:27.974 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 44406326~5644129 2024-04-26T04:52:27.987 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 49706380~3430554 2024-04-26T04:52:27.989 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:52:28.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:27 smithi005 ceph-mon[95809]: pgmap v221: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-04-26T04:52:28.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:27 smithi005 ceph-mon[95809]: osdmap e401: 8 total, 8 up, 8 in 2024-04-26T04:52:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:27 smithi005 ceph-mon[91801]: pgmap v221: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-04-26T04:52:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:27 smithi005 ceph-mon[91801]: osdmap e401: 8 total, 8 up, 8 in 2024-04-26T04:52:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:27 smithi111 ceph-mon[30255]: pgmap v221: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-04-26T04:52:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:27 smithi111 ceph-mon[30255]: osdmap e401: 8 total, 8 up, 8 in 2024-04-26T04:52:28.861 INFO:tasks.workunit.client.0.smithi005.stdout:clone image241 -> image242 2024-04-26T04:52:28.928 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 55788160~4468609 2024-04-26T04:52:28.964 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 9596497~8214607 2024-04-26T04:52:28.968 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 58963266~2047856 2024-04-26T04:52:28.977 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 17649137~6134151 2024-04-26T04:52:28.981 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 24867600~6913023 2024-04-26T04:52:28.995 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 35583949~4246287 2024-04-26T04:52:28.999 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 17514971~1003012 2024-04-26T04:52:29.002 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 649636~576800 2024-04-26T04:52:29.005 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:52:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:28 smithi111 ceph-mon[30255]: osdmap e402: 8 total, 8 up, 8 in 2024-04-26T04:52:29.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:52:28 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:52:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:52:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:28 smithi005 ceph-mon[91801]: osdmap e402: 8 total, 8 up, 8 in 2024-04-26T04:52:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:28 smithi005 ceph-mon[95809]: osdmap e402: 8 total, 8 up, 8 in 2024-04-26T04:52:29.871 INFO:tasks.workunit.client.0.smithi005.stdout:clone image242 -> image243 2024-04-26T04:52:29.954 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 139727212 2024-04-26T04:52:29.958 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 73383151~272705 2024-04-26T04:52:29.965 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 33816166~6447050 2024-04-26T04:52:29.970 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 115381924~7328429 2024-04-26T04:52:29.990 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 38334191~8287460 2024-04-26T04:52:30.006 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 109071780~4002327 2024-04-26T04:52:30.027 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 38568015~4125625 2024-04-26T04:52:30.028 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 35494254~3742193 2024-04-26T04:52:30.036 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 67046990~1960459 2024-04-26T04:52:30.068 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap3 2024-04-26T04:52:30.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:29 smithi005 ceph-mon[91801]: pgmap v224: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 41 MiB/s wr, 584 op/s 2024-04-26T04:52:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:29 smithi005 ceph-mon[91801]: osdmap e403: 8 total, 8 up, 8 in 2024-04-26T04:52:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:29 smithi005 ceph-mon[95809]: pgmap v224: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 41 MiB/s wr, 584 op/s 2024-04-26T04:52:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:29 smithi005 ceph-mon[95809]: osdmap e403: 8 total, 8 up, 8 in 2024-04-26T04:52:30.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:29 smithi111 ceph-mon[30255]: pgmap v224: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 41 MiB/s wr, 584 op/s 2024-04-26T04:52:30.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:29 smithi111 ceph-mon[30255]: osdmap e403: 8 total, 8 up, 8 in 2024-04-26T04:52:30.874 INFO:tasks.workunit.client.0.smithi005.stdout:clone image243 -> image244 2024-04-26T04:52:30.976 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 115124182 2024-04-26T04:52:31.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:30 smithi111 ceph-mon[30255]: osdmap e404: 8 total, 8 up, 8 in 2024-04-26T04:52:31.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:30 smithi005 ceph-mon[91801]: osdmap e404: 8 total, 8 up, 8 in 2024-04-26T04:52:31.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:30 smithi005 ceph-mon[95809]: osdmap e404: 8 total, 8 up, 8 in 2024-04-26T04:52:31.239 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Stress (3947 ms) 2024-04-26T04:52:31.239 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-04-26T04:52:31.492 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (253 ms) 2024-04-26T04:52:31.492 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-04-26T04:52:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:31 smithi005 ceph-mon[91801]: pgmap v227: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 41 MiB/s wr, 584 op/s 2024-04-26T04:52:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:31 smithi005 ceph-mon[91801]: osdmap e405: 8 total, 8 up, 8 in 2024-04-26T04:52:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:31 smithi005 ceph-mon[95809]: pgmap v227: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 41 MiB/s wr, 584 op/s 2024-04-26T04:52:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:31 smithi005 ceph-mon[95809]: osdmap e405: 8 total, 8 up, 8 in 2024-04-26T04:52:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:31 smithi111 ceph-mon[30255]: pgmap v227: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 41 MiB/s wr, 584 op/s 2024-04-26T04:52:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:31 smithi111 ceph-mon[30255]: osdmap e405: 8 total, 8 up, 8 in 2024-04-26T04:52:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:31 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:33.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:32 smithi005 ceph-mon[91801]: osdmap e406: 8 total, 8 up, 8 in 2024-04-26T04:52:33.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:32 smithi005 ceph-mon[95809]: osdmap e406: 8 total, 8 up, 8 in 2024-04-26T04:52:33.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:32 smithi111 ceph-mon[30255]: osdmap e406: 8 total, 8 up, 8 in 2024-04-26T04:52:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:52:33] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2024-04-26T04:52:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:33 smithi005 ceph-mon[91801]: pgmap v230: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:52:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:33 smithi005 ceph-mon[91801]: osdmap e407: 8 total, 8 up, 8 in 2024-04-26T04:52:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:33 smithi005 ceph-mon[95809]: pgmap v230: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:52:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:33 smithi005 ceph-mon[95809]: osdmap e407: 8 total, 8 up, 8 in 2024-04-26T04:52:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:33 smithi111 ceph-mon[30255]: pgmap v230: 105 pgs: 105 active+clean; 222 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:52:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:33 smithi111 ceph-mon[30255]: osdmap e407: 8 total, 8 up, 8 in 2024-04-26T04:52:35.179 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (3687 ms) 2024-04-26T04:52:35.179 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-04-26T04:52:35.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:34 smithi005 ceph-mon[91801]: osdmap e408: 8 total, 8 up, 8 in 2024-04-26T04:52:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:34 smithi005 ceph-mon[95809]: osdmap e408: 8 total, 8 up, 8 in 2024-04-26T04:52:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:34 smithi111 ceph-mon[30255]: osdmap e408: 8 total, 8 up, 8 in 2024-04-26T04:52:36.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:35 smithi005 ceph-mon[95809]: pgmap v233: 105 pgs: 105 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 28 MiB/s wr, 724 op/s 2024-04-26T04:52:36.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:35 smithi005 ceph-mon[95809]: osdmap e409: 8 total, 8 up, 8 in 2024-04-26T04:52:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:35 smithi005 ceph-mon[91801]: pgmap v233: 105 pgs: 105 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 28 MiB/s wr, 724 op/s 2024-04-26T04:52:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:35 smithi005 ceph-mon[91801]: osdmap e409: 8 total, 8 up, 8 in 2024-04-26T04:52:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:35 smithi111 ceph-mon[30255]: pgmap v233: 105 pgs: 105 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 28 MiB/s wr, 724 op/s 2024-04-26T04:52:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:35 smithi111 ceph-mon[30255]: osdmap e409: 8 total, 8 up, 8 in 2024-04-26T04:52:37.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:36 smithi005 ceph-mon[91801]: osdmap e410: 8 total, 8 up, 8 in 2024-04-26T04:52:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:36 smithi005 ceph-mon[95809]: osdmap e410: 8 total, 8 up, 8 in 2024-04-26T04:52:37.249 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2070 ms) 2024-04-26T04:52:37.249 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-04-26T04:52:37.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:36 smithi111 ceph-mon[30255]: osdmap e410: 8 total, 8 up, 8 in 2024-04-26T04:52:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:37 smithi005 ceph-mon[91801]: pgmap v236: 105 pgs: 105 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 28 MiB/s wr, 724 op/s 2024-04-26T04:52:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:37 smithi005 ceph-mon[91801]: osdmap e411: 8 total, 8 up, 8 in 2024-04-26T04:52:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:37 smithi005 ceph-mon[95809]: pgmap v236: 105 pgs: 105 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 28 MiB/s wr, 724 op/s 2024-04-26T04:52:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:37 smithi005 ceph-mon[95809]: osdmap e411: 8 total, 8 up, 8 in 2024-04-26T04:52:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:37 smithi111 ceph-mon[30255]: pgmap v236: 105 pgs: 105 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 28 MiB/s wr, 724 op/s 2024-04-26T04:52:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:37 smithi111 ceph-mon[30255]: osdmap e411: 8 total, 8 up, 8 in 2024-04-26T04:52:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:38 smithi111 ceph-mon[30255]: osdmap e412: 8 total, 8 up, 8 in 2024-04-26T04:52:39.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:52:38 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:52:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:52:39.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:38 smithi005 ceph-mon[91801]: osdmap e412: 8 total, 8 up, 8 in 2024-04-26T04:52:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:38 smithi005 ceph-mon[95809]: osdmap e412: 8 total, 8 up, 8 in 2024-04-26T04:52:39.361 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2112 ms) 2024-04-26T04:52:39.361 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-04-26T04:52:39.382 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 36311833~8359594 2024-04-26T04:52:39.408 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 34263173~3400415 2024-04-26T04:52:39.409 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 45993091~3334202 2024-04-26T04:52:39.422 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 75676208~8209872 2024-04-26T04:52:39.422 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 8482578~1170928 2024-04-26T04:52:39.428 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 66338234~849378 2024-04-26T04:52:39.428 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 8177808~2737300 2024-04-26T04:52:39.436 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 75696329~8189751 2024-04-26T04:52:39.437 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap0 2024-04-26T04:52:39.926 INFO:tasks.workunit.client.0.smithi005.stdout:clone image263 -> image264 2024-04-26T04:52:39.974 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 30483757~1909736 2024-04-26T04:52:39.986 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 47594969~5937929 2024-04-26T04:52:40.001 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 7898388~6511463 2024-04-26T04:52:40.025 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 26530065~2213404 2024-04-26T04:52:40.029 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 38463462~1713249 2024-04-26T04:52:40.053 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 61161389~5477549 2024-04-26T04:52:40.057 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 22225751~7306432 2024-04-26T04:52:40.074 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 61038495~5198535 2024-04-26T04:52:40.075 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:52:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:39 smithi005 ceph-mon[91801]: pgmap v239: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 381 op/s 2024-04-26T04:52:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:39 smithi005 ceph-mon[91801]: osdmap e413: 8 total, 8 up, 8 in 2024-04-26T04:52:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:39 smithi005 ceph-mon[95809]: pgmap v239: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 381 op/s 2024-04-26T04:52:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:39 smithi005 ceph-mon[95809]: osdmap e413: 8 total, 8 up, 8 in 2024-04-26T04:52:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:39 smithi111 ceph-mon[30255]: pgmap v239: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 381 op/s 2024-04-26T04:52:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:39 smithi111 ceph-mon[30255]: osdmap e413: 8 total, 8 up, 8 in 2024-04-26T04:52:40.930 INFO:tasks.workunit.client.0.smithi005.stdout:clone image264 -> image265 2024-04-26T04:52:40.988 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 107031587 2024-04-26T04:52:40.993 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 16830192~5978541 2024-04-26T04:52:41.021 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 13539099~7539219 2024-04-26T04:52:41.029 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 26971759~1824374 2024-04-26T04:52:41.051 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 3019211~421204 2024-04-26T04:52:41.054 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 100461855~6569732 2024-04-26T04:52:41.071 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 102033210~2805882 2024-04-26T04:52:41.073 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 28941665~4967093 2024-04-26T04:52:41.095 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 33457189~7922524 2024-04-26T04:52:41.104 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:52:41.179 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:40 smithi005 ceph-mon[91801]: osdmap e414: 8 total, 8 up, 8 in 2024-04-26T04:52:41.179 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:41.179 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:40 smithi005 ceph-mon[95809]: osdmap e414: 8 total, 8 up, 8 in 2024-04-26T04:52:41.180 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:40 smithi111 ceph-mon[30255]: osdmap e414: 8 total, 8 up, 8 in 2024-04-26T04:52:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:40 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:41.935 INFO:tasks.workunit.client.0.smithi005.stdout:clone image265 -> image266 2024-04-26T04:52:42.008 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 99270919~4443818 2024-04-26T04:52:42.022 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 58455136~756946 2024-04-26T04:52:42.026 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 5218396~4699852 2024-04-26T04:52:42.047 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 98760914~8270673 2024-04-26T04:52:42.053 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 61711517~8006 2024-04-26T04:52:42.054 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 89484486~7917057 2024-04-26T04:52:42.057 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 89085730~179987 2024-04-26T04:52:42.058 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 29942693~1127131 2024-04-26T04:52:42.082 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap3 2024-04-26T04:52:42.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:41 smithi005 ceph-mon[91801]: pgmap v242: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 381 op/s 2024-04-26T04:52:42.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:41 smithi005 ceph-mon[91801]: osdmap e415: 8 total, 8 up, 8 in 2024-04-26T04:52:42.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:41 smithi005 ceph-mon[95809]: pgmap v242: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 381 op/s 2024-04-26T04:52:42.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:41 smithi005 ceph-mon[95809]: osdmap e415: 8 total, 8 up, 8 in 2024-04-26T04:52:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:41 smithi111 ceph-mon[30255]: pgmap v242: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 381 op/s 2024-04-26T04:52:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:41 smithi111 ceph-mon[30255]: osdmap e415: 8 total, 8 up, 8 in 2024-04-26T04:52:42.373 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:52:42.941 INFO:tasks.workunit.client.0.smithi005.stdout:clone image266 -> image267 2024-04-26T04:52:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:42 smithi111 ceph-mon[30255]: osdmap e416: 8 total, 8 up, 8 in 2024-04-26T04:52:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:42 smithi005 ceph-mon[91801]: osdmap e416: 8 total, 8 up, 8 in 2024-04-26T04:52:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:42 smithi005 ceph-mon[95809]: osdmap e416: 8 total, 8 up, 8 in 2024-04-26T04:52:43.246 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (3885 ms) 2024-04-26T04:52:43.246 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-04-26T04:52:43.332 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:43.460 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:52:43] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2024-04-26T04:52:43.484 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (238 ms) 2024-04-26T04:52:43.484 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-04-26T04:52:43.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:43 smithi005 ceph-mon[91801]: pgmap v245: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:52:43.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:43 smithi005 ceph-mon[91801]: osdmap e417: 8 total, 8 up, 8 in 2024-04-26T04:52:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:43 smithi005 ceph-mon[95809]: pgmap v245: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:52:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:43 smithi005 ceph-mon[95809]: osdmap e417: 8 total, 8 up, 8 in 2024-04-26T04:52:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:43 smithi111 ceph-mon[30255]: pgmap v245: 105 pgs: 105 active+clean; 435 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:52:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:43 smithi111 ceph-mon[30255]: osdmap e417: 8 total, 8 up, 8 in 2024-04-26T04:52:45.201 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:44 smithi005 ceph-mon[91801]: osdmap e418: 8 total, 8 up, 8 in 2024-04-26T04:52:45.202 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:44 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:45.202 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:44 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:45.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:44 smithi005 ceph-mon[95809]: osdmap e418: 8 total, 8 up, 8 in 2024-04-26T04:52:45.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:44 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:45.202 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:44 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:44 smithi111 ceph-mon[30255]: osdmap e418: 8 total, 8 up, 8 in 2024-04-26T04:52:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:44 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:44 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:46.052 INFO:teuthology.orchestra.run.smithi005.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:52:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:45 smithi005 ceph-mon[91801]: pgmap v248: 105 pgs: 105 active+clean; 626 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 42 MiB/s wr, 562 op/s 2024-04-26T04:52:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:45 smithi005 ceph-mon[91801]: osdmap e419: 8 total, 8 up, 8 in 2024-04-26T04:52:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:45 smithi005 ceph-mon[95809]: pgmap v248: 105 pgs: 105 active+clean; 626 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 42 MiB/s wr, 562 op/s 2024-04-26T04:52:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:45 smithi005 ceph-mon[95809]: osdmap e419: 8 total, 8 up, 8 in 2024-04-26T04:52:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:45 smithi111 ceph-mon[30255]: pgmap v248: 105 pgs: 105 active+clean; 626 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 42 MiB/s wr, 562 op/s 2024-04-26T04:52:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:45 smithi111 ceph-mon[30255]: osdmap e419: 8 total, 8 up, 8 in 2024-04-26T04:52:46.739 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for all mons to upgrade' 2024-04-26T04:52:46.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[91801]: osdmap e420: 8 total, 8 up, 8 in 2024-04-26T04:52:46.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[91801]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:46.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[91801]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:52:46.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:46.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:46.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[95809]: osdmap e420: 8 total, 8 up, 8 in 2024-04-26T04:52:46.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[95809]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:46.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[95809]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:52:46.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:46.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:47.179 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (3695 ms) 2024-04-26T04:52:47.180 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-04-26T04:52:47.212 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:46 smithi111 ceph-mon[30255]: osdmap e420: 8 total, 8 up, 8 in 2024-04-26T04:52:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:46 smithi111 ceph-mon[30255]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:46 smithi111 ceph-mon[30255]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:52:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:46 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:46 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:48.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[95809]: pgmap v251: 105 pgs: 105 active+clean; 626 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 42 MiB/s wr, 562 op/s 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[95809]: osdmap e421: 8 total, 8 up, 8 in 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[91801]: pgmap v251: 105 pgs: 105 active+clean; 626 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 42 MiB/s wr, 562 op/s 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[91801]: osdmap e421: 8 total, 8 up, 8 in 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:47 smithi111 ceph-mon[30255]: pgmap v251: 105 pgs: 105 active+clean; 626 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 42 MiB/s wr, 562 op/s 2024-04-26T04:52:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:47 smithi111 ceph-mon[30255]: osdmap e421: 8 total, 8 up, 8 in 2024-04-26T04:52:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:47 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:47 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:48.782 INFO:teuthology.orchestra.run.smithi005.stdout:wait for all mons to upgrade 2024-04-26T04:52:49.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:52:48 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:52:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:52:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: pgmap v253: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s rd, 19 MiB/s wr, 493 op/s 2024-04-26T04:52:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: osdmap e422: 8 total, 8 up, 8 in 2024-04-26T04:52:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: pgmap v255: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 22 MiB/s wr, 589 op/s 2024-04-26T04:52:49.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: pgmap v256: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s rd, 31 MiB/s wr, 819 op/s 2024-04-26T04:52:49.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:49.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:49.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:52:49.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:52:49.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:49 smithi111 ceph-mon[30255]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: pgmap v253: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s rd, 19 MiB/s wr, 493 op/s 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: osdmap e422: 8 total, 8 up, 8 in 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: pgmap v255: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 22 MiB/s wr, 589 op/s 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: pgmap v256: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s rd, 31 MiB/s wr, 819 op/s 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.520 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[95809]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: pgmap v253: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s rd, 19 MiB/s wr, 493 op/s 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: osdmap e422: 8 total, 8 up, 8 in 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: pgmap v255: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 22 MiB/s wr, 589 op/s 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: pgmap v256: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s rd, 31 MiB/s wr, 819 op/s 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.521 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:52:49.522 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:52:49.522 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:49 smithi005 ceph-mon[91801]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:52:49.533 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2354 ms) 2024-04-26T04:52:49.533 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-04-26T04:52:49.551 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-26T04:52:49.823 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:50.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:50 smithi005 ceph-mon[91801]: osdmap e423: 8 total, 8 up, 8 in 2024-04-26T04:52:50.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:50 smithi005 ceph-mon[95809]: osdmap e423: 8 total, 8 up, 8 in 2024-04-26T04:52:50.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:50 smithi111 ceph-mon[30255]: osdmap e423: 8 total, 8 up, 8 in 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: osdmap e424: 8 total, 8 up, 8 in 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: Upgrade: Updating mon.b 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: Deploying daemon mon.b on smithi111 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[95809]: pgmap v259: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:52:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: osdmap e424: 8 total, 8 up, 8 in 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: Upgrade: Updating mon.b 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: Deploying daemon mon.b on smithi111 2024-04-26T04:52:51.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:51 smithi005 ceph-mon[91801]: pgmap v259: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:52:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: osdmap e424: 8 total, 8 up, 8 in 2024-04-26T04:52:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: Upgrade: Updating mon.b 2024-04-26T04:52:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:51.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:52:51.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:52:51.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:51.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: Deploying daemon mon.b on smithi111 2024-04-26T04:52:51.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:51 smithi111 ceph-mon[30255]: pgmap v259: 105 pgs: 105 active+clean; 706 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:52:51.781 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2248 ms) 2024-04-26T04:52:51.781 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-04-26T04:52:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 65389710~4280312 2024-04-26T04:52:51.820 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 23042123~7370962 2024-04-26T04:52:51.820 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 32271993~3247939 2024-04-26T04:52:51.835 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 55810002~5798823 2024-04-26T04:52:51.835 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 13231127~4964231 2024-04-26T04:52:51.850 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 71295399~1533564 2024-04-26T04:52:51.852 INFO:tasks.workunit.client.0.smithi005.stdout:write: A 69606669~5599465 2024-04-26T04:52:51.868 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 8797545~497779 2024-04-26T04:52:51.869 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap0 2024-04-26T04:52:52.358 INFO:tasks.workunit.client.0.smithi005.stdout:clone image286 -> image287 2024-04-26T04:52:52.410 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 84303023 2024-04-26T04:52:52.415 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 systemd[1]: Stopping Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:52:52.416 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 ceph-mon[30255]: osdmap e425: 8 total, 8 up, 8 in 2024-04-26T04:52:52.416 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 ceph-mon[30255]: from='client.? 172.21.15.5:0/2517797984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:52:52.417 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 51700713~4580174 2024-04-26T04:52:52.437 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 71035140~569079 2024-04-26T04:52:52.473 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 40720649~7292999 2024-04-26T04:52:52.492 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 67953320~4090760 2024-04-26T04:52:52.494 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 2268031~4647716 2024-04-26T04:52:52.505 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 67565858~5707383 2024-04-26T04:52:52.507 INFO:tasks.workunit.client.0.smithi005.stdout:write: B 60623770~6081653 2024-04-26T04:52:52.530 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 45116854~8195151 2024-04-26T04:52:52.532 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:52:52.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-mon[91801]: osdmap e425: 8 total, 8 up, 8 in 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2517797984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-mon[95809]: osdmap e425: 8 total, 8 up, 8 in 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2517797984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:52:52] ENGINE Bus STOPPING 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:52:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:52:52] ENGINE Bus STOPPED 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:52:52] ENGINE Bus STARTING 2024-04-26T04:52:52.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:52:52] ENGINE Serving on http://:::9283 2024-04-26T04:52:52.731 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:52:52] ENGINE Bus STARTED 2024-04-26T04:52:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b[30238]: 2024-04-26T04:52:52.413+0000 7f59595f4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:52:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b[30238]: 2024-04-26T04:52:52.413+0000 7f59595f4700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T04:52:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 podman[86197]: 2024-04-26 04:52:52.495554912 +0000 UTC m=+0.205611907 container died 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T04:52:53.124 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 podman[86197]: 2024-04-26 04:52:52.881105121 +0000 UTC m=+0.591162103 container cleanup 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T04:52:53.124 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:52 smithi111 bash[86197]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b 2024-04-26T04:52:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:52:53] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2024-04-26T04:52:53.495 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 podman[86211]: 2024-04-26 04:52:53.124189323 +0000 UTC m=+0.623485052 container remove 20923c1c0e0380134b02ca839dc4e2e52976b1249c759a340a270b0e4978ed82 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c) 2024-04-26T04:52:53.495 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-26T04:52:53.495 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 systemd[1]: Stopped Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:52:53.495 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.b.service: Consumed 11.102s CPU time. 2024-04-26T04:52:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 systemd[1]: Starting Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:52:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 podman[86341]: 2024-04-26 04:52:53.611919548 +0000 UTC m=+0.017676810 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:52:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:53 smithi111 podman[86341]: 2024-04-26 04:52:53.745071823 +0000 UTC m=+0.150829076 container create c986c5c74b8097a4ccf42367789d100f01ad42fa8da95fadad14a42ed679ad76 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T04:52:54.145 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 podman[86341]: 2024-04-26 04:52:54.069406132 +0000 UTC m=+0.475163379 container init c986c5c74b8097a4ccf42367789d100f01ad42fa8da95fadad14a42ed679ad76 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T04:52:54.145 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 podman[86341]: 2024-04-26 04:52:54.072503667 +0000 UTC m=+0.478260915 container start c986c5c74b8097a4ccf42367789d100f01ad42fa8da95fadad14a42ed679ad76 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-26T04:52:54.145 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:52:54.145 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: pidfile_write: ignore empty --pid-file 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: load: jerasure load: lrc 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Git sha 0 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: DB SUMMARY 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: DB Session ID: YW00LCGF0163K6ZTDYJ1 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1295 Bytes 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000030.sst 2024-04-26T04:52:54.146 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000028.log size: 1513586 ; 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.env: 0x55b1aba7bc60 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.info_log: 0x55b1ad445aa0 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.statistics: (nil) 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.use_fsync: 0 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:52:54.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.db_log_dir: 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.wal_dir: 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:52:54.148 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.write_buffer_manager: 0x55b1ad4181e0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:52:54.149 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.unordered_write: 0 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.row_cache: None 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.wal_filter: None 2024-04-26T04:52:54.150 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:52:54.151 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:52:54.151 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:52:54.151 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:52:54.151 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.wal_compression: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:52:54.152 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_open_files: -1 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:52:54.153 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Compression algorithms supported: 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kZSTD supported: 0 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:52:54.154 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.merge_operator: 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_filter: None 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b1ad445ba0) 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: index_type: 0 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: data_block_index_type: 0 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: index_shortening: 1 2024-04-26T04:52:54.155 INFO:journalctl@ceph.mon.b.smithi111.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: checksum: 4 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: no_block_cache: 0 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache: 0x55b1ad449090 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache_options: 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: capacity : 536870912 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: num_shard_bits : 4 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: strict_capacity_limit : 0 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_cache_compressed: (nil) 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: persistent_cache: (nil) 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_size: 4096 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_size_deviation: 10 2024-04-26T04:52:54.156 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_restart_interval: 16 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: index_block_restart_interval: 1 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: metadata_block_size: 4096 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: partition_filters: 0 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: use_delta_encoding: 1 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: filter_policy: bloomfilter 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: whole_key_filtering: 1 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: verify_compression: 0 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: format_version: 5 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: enable_index_compression: 1 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: block_align: 0 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: max_auto_readahead_size: 262144 2024-04-26T04:52:54.157 INFO:journalctl@ceph.mon.b.smithi111.stdout: prepopulate_block_cache: 0 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression: NoCompression 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.num_levels: 7 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:52:54.158 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:52:54.159 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:52:54.160 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:52:54.161 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:52:54.162 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.ttl: 2592000 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:52:54.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 32, last_sequence is 19331, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6e051ac3-38ed-4fe1-bf01-780d72ab17ee 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107174101763, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-26T04:52:54.164 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107174106188, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 995028, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19336, "largest_seqno": 20133, "table_properties": {"data_size": 990423, "index_size": 2823, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 6708, "raw_average_key_size": 22, "raw_value_size": 983133, "raw_average_value_size": 3288, "num_data_blocks": 122, "num_entries": 299, "num_filter_entries": 299, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107174, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6e051ac3-38ed-4fe1-bf01-780d72ab17ee", "db_session_id": "YW00LCGF0163K6ZTDYJ1", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107174106254, "job": 1, "event": "recovery_finished"} 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b1ad53a000 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: DB pointer 0x55b1ad524000 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** DB Stats ** 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:52:54.165 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** Compaction Stats [default] ** 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: L0 1/0 971.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 416.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: L6 1/0 12.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: Sum 2/0 13.72 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 416.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 416.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** Compaction Stats [default] ** 2024-04-26T04:52:54.166 INFO:journalctl@ceph.mon.b.smithi111.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 416.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: Cumulative compaction: 0.00 GB write, 139.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: Interval compaction: 0.00 GB write, 139.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:52:54.167 INFO:journalctl@ceph.mon.b.smithi111.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: Block cache BinnedLRUCache@0x55b1ad449090#2 capacity: 512.00 MB usage: 3.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: starting mon.b rank 1 at public addrs [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] at bind addrs [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???) e3 preinit fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).mds e9 new map 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).mds e9 print_map 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: e9 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: legacy client fscid: 1 2024-04-26T04:52:54.168 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: Filesystem 'foo' (1) 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: fs_name foo 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: epoch 8 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: created 2024-04-26T04:45:21.622880+0000 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: modified 2024-04-26T04:45:39.686257+0000 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: tableserver 0 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: root 0 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: session_timeout 60 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: session_autoclose 300 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: max_file_size 1099511627776 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: max_xattr_size 65536 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: required_client_features {} 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: last_failure 0 2024-04-26T04:52:54.169 INFO:journalctl@ceph.mon.b.smithi111.stdout: last_failure_osd_epoch 0 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: max_mds 1 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: in 0 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: up {0=24554} 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: failed 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: damaged 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: stopped 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: data_pools [4] 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: metadata_pool 3 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: inline_data disabled 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: balancer 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: bal_rank_mask -1 2024-04-26T04:52:54.170 INFO:journalctl@ceph.mon.b.smithi111.stdout: standby_count_wanted 1 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: [mds.foo.smithi005.zdnlsb{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.5:6834/2816589462,v1:172.21.15.5:6835/2816589462] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: Standby daemons: 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout: [mds.foo.smithi111.furhfd{-1:24538} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.111:6832/928927067,v1:172.21.15.111:6833/928927067] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).osd e426 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).osd e426 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).osd e426 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).osd e426 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 ceph-mon[86360]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 bash[86341]: c986c5c74b8097a4ccf42367789d100f01ad42fa8da95fadad14a42ed679ad76 2024-04-26T04:52:54.171 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:54 smithi111 systemd[1]: Started Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:52:55.577 INFO:tasks.workunit.client.0.smithi005.stdout:clone image287 -> image288 2024-04-26T04:52:55.640 INFO:tasks.workunit.client.0.smithi005.stdout:resize: 140222016 2024-04-26T04:52:55.646 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 135234768~4325915 2024-04-26T04:52:55.666 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 36855639~6210686 2024-04-26T04:52:55.697 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 64526120~7674871 2024-04-26T04:52:55.729 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 131530475~8357284 2024-04-26T04:52:55.733 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 45353282~3830481 2024-04-26T04:52:55.743 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 77273600~2041602 2024-04-26T04:52:55.747 INFO:tasks.workunit.client.0.smithi005.stdout:write: C 115263835~6928721 2024-04-26T04:52:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: pgmap v263: 105 pgs: 105 active+clean; 887 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 32 MiB/s wr, 520 op/s 2024-04-26T04:52:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:52:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:52:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: osdmap e426: 8 total, 8 up, 8 in 2024-04-26T04:52:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: mgrmap e38: y(active, since 2m), standbys: x 2024-04-26T04:52:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: mon.b calling monitor election 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: mon.a calling monitor election 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: mon.c calling monitor election 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: monmap epoch 4 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: last_changed 2024-04-26T04:52:54.544589+0000 2024-04-26T04:52:55.769 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: min_mon_release 19 (squid) 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: election_strategy: 1 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: osdmap e426: 8 total, 8 up, 8 in 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: mgrmap e38: y(active, since 2m), standbys: x 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='' 2024-04-26T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:55.771 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:55 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:55.771 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 15221875~4816503 2024-04-26T04:52:55.789 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: pgmap v263: 105 pgs: 105 active+clean; 887 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 32 MiB/s wr, 520 op/s 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: osdmap e426: 8 total, 8 up, 8 in 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: mgrmap e38: y(active, since 2m), standbys: x 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:52:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: mon.b calling monitor election 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: mon.a calling monitor election 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: mon.c calling monitor election 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: monmap epoch 4 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: last_changed 2024-04-26T04:52:54.544589+0000 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: min_mon_release 19 (squid) 2024-04-26T04:52:55.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: election_strategy: 1 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: osdmap e426: 8 total, 8 up, 8 in 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: mgrmap e38: y(active, since 2m), standbys: x 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='' 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:55.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: pgmap v263: 105 pgs: 105 active+clean; 887 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 32 MiB/s wr, 520 op/s 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: osdmap e426: 8 total, 8 up, 8 in 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: mgrmap e38: y(active, since 2m), standbys: x 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: mon.b calling monitor election 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: mon.a calling monitor election 2024-04-26T04:52:55.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: mon.c calling monitor election 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: monmap epoch 4 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: last_changed 2024-04-26T04:52:54.544589+0000 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: created 2024-04-26T04:38:19.110602+0000 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: min_mon_release 19 (squid) 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: election_strategy: 1 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: 0: [v2:172.21.15.5:3300/0,v1:172.21.15.5:6789/0] mon.a 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: 1: [v2:172.21.15.111:3300/0,v1:172.21.15.111:6789/0] mon.b 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: 2: [v2:172.21.15.5:3301/0,v1:172.21.15.5:6790/0] mon.c 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: fsmap foo:1 {0=foo.smithi005.zdnlsb=up:active} 1 up:standby 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: osdmap e426: 8 total, 8 up, 8 in 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: mgrmap e38: y(active, since 2m), standbys: x 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:52:55.984 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:52:55.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: daemon grafana.a on smithi111 is in error state 2024-04-26T04:52:55.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='' 2024-04-26T04:52:55.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:55.985 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:55 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:56.577 INFO:tasks.workunit.client.0.smithi005.stdout:clone image288 -> image289 2024-04-26T04:52:56.646 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 107787927~7391035 2024-04-26T04:52:56.671 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 77362936~2771822 2024-04-26T04:52:56.675 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 53927659~2146062 2024-04-26T04:52:56.702 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 83929107~6885421 2024-04-26T04:52:56.705 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 28682285~3402555 2024-04-26T04:52:56.725 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 125197508~4981852 2024-04-26T04:52:56.732 INFO:tasks.workunit.client.0.smithi005.stdout:write: D 66792375~3822932 2024-04-26T04:52:56.741 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T04:52:56.767 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 91448402~5414996 2024-04-26T04:52:56.770 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap3 2024-04-26T04:52:56.957 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:56 smithi111 ceph-mon[86360]: osdmap e427: 8 total, 8 up, 8 in 2024-04-26T04:52:56.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:56 smithi005 ceph-mon[91801]: osdmap e427: 8 total, 8 up, 8 in 2024-04-26T04:52:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:56 smithi005 ceph-mon[95809]: osdmap e427: 8 total, 8 up, 8 in 2024-04-26T04:52:57.345 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:52:57.582 INFO:tasks.workunit.client.0.smithi005.stdout:clone image289 -> image290 2024-04-26T04:52:57.621 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:57.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:57 smithi005 ceph-mon[91801]: pgmap v265: 105 pgs: 105 active+clean; 887 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 28 MiB/s wr, 457 op/s 2024-04-26T04:52:57.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:57 smithi005 ceph-mon[91801]: osdmap e428: 8 total, 8 up, 8 in 2024-04-26T04:52:57.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:57 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2691906163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:52:57.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:57 smithi005 ceph-mon[95809]: pgmap v265: 105 pgs: 105 active+clean; 887 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 28 MiB/s wr, 457 op/s 2024-04-26T04:52:57.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:57 smithi005 ceph-mon[95809]: osdmap e428: 8 total, 8 up, 8 in 2024-04-26T04:52:57.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:57 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2691906163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:52:57.958 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:57 smithi111 ceph-mon[86360]: pgmap v265: 105 pgs: 105 active+clean; 887 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 28 MiB/s wr, 457 op/s 2024-04-26T04:52:57.958 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:57 smithi111 ceph-mon[86360]: osdmap e428: 8 total, 8 up, 8 in 2024-04-26T04:52:57.958 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:57 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2691906163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:52:57.979 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (6198 ms) 2024-04-26T04:52:57.979 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-04-26T04:52:58.001 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.073 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (94 ms) 2024-04-26T04:52:58.073 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-04-26T04:52:58.094 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.161 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (88 ms) 2024-04-26T04:52:58.162 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-04-26T04:52:58.183 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.251 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (90 ms) 2024-04-26T04:52:58.252 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-04-26T04:52:58.272 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.338 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (87 ms) 2024-04-26T04:52:58.339 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-04-26T04:52:58.360 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.429 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (91 ms) 2024-04-26T04:52:58.429 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-04-26T04:52:58.451 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.522 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (93 ms) 2024-04-26T04:52:58.523 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-04-26T04:52:58.545 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.614 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (91 ms) 2024-04-26T04:52:58.614 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-04-26T04:52:58.635 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.704 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (90 ms) 2024-04-26T04:52:58.705 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-04-26T04:52:58.732 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.803 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (99 ms) 2024-04-26T04:52:58.803 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-04-26T04:52:58.825 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:52:58.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:58 smithi005 ceph-mon[95809]: osdmap e429: 8 total, 8 up, 8 in 2024-04-26T04:52:58.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:58 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:58.829 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:58 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:58.829 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:58 smithi005 ceph-mon[91801]: osdmap e429: 8 total, 8 up, 8 in 2024-04-26T04:52:58.829 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:58 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:58.829 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:58 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:58.899 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (96 ms) 2024-04-26T04:52:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:58 smithi111 ceph-mon[86360]: osdmap e429: 8 total, 8 up, 8 in 2024-04-26T04:52:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:58 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:58 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:52:59.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:52:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:52:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:52:59.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:52:59 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:52:59.113+0000 7f15dde08700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:alertmanager.a smithi005 *:9093,9094 running (4m) 12s ago 8m 25.1M - 0.25.0 c8568f914cd2 3afeca478730 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:grafana.a smithi111 *:3000 error 1s ago 8m - - 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi005.zdnlsb smithi005 running (7m) 12s ago 7m 20.2M - 17.2.7-902-gc61472c8 0de407c1b3af 7ddb93a6b5a1 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi111.furhfd smithi111 running (7m) 1s ago 7m 17.2M - 17.2.7-902-gc61472c8 0de407c1b3af 3e206ca72f2e 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.x smithi111 *:8443,9283,8765 running (3m) 1s ago 12m 450M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0572da61dd7 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.y smithi005 *:8443,9283,8765 running (3m) 12s ago 14m 502M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c895550b1a50 2024-04-26T04:52:59.556 INFO:teuthology.orchestra.run.smithi005.stdout:mon.a smithi005 running (3m) 12s ago 14m 48.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 249028d8f146 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:mon.b smithi111 running (5s) 1s ago 13m 21.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c986c5c74b80 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:mon.c smithi005 running (92s) 12s ago 12m 38.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9834a90fbbaa 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.a smithi005 *:9100 running (4m) 12s ago 8m 16.7M - 1.5.0 0da6a335fe13 c74ab514255d 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.b smithi111 *:9100 running (4m) 1s ago 8m 17.3M - 1.5.0 0da6a335fe13 cc0664bb12c9 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:osd.0 smithi005 running (12m) 12s ago 12m 119M 2481M 17.2.7-902-gc61472c8 0de407c1b3af b99cc094c6d7 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:osd.1 smithi005 running (11m) 12s ago 11m 121M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 9e578b598142 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:osd.2 smithi005 running (11m) 12s ago 11m 154M 2481M 17.2.7-902-gc61472c8 0de407c1b3af f25e1cd874ca 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:osd.3 smithi005 running (11m) 12s ago 11m 128M 2481M 17.2.7-902-gc61472c8 0de407c1b3af fd23fb067fc9 2024-04-26T04:52:59.557 INFO:teuthology.orchestra.run.smithi005.stdout:osd.4 smithi111 running (10m) 1s ago 10m 136M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e2994c2a7aa2 2024-04-26T04:52:59.558 INFO:teuthology.orchestra.run.smithi005.stdout:osd.5 smithi111 running (10m) 1s ago 10m 99.2M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5da203330767 2024-04-26T04:52:59.558 INFO:teuthology.orchestra.run.smithi005.stdout:osd.6 smithi111 running (9m) 1s ago 9m 140M 2480M 17.2.7-902-gc61472c8 0de407c1b3af b7798f2bbff0 2024-04-26T04:52:59.558 INFO:teuthology.orchestra.run.smithi005.stdout:osd.7 smithi111 running (9m) 1s ago 9m 115M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 60f1bafb36a6 2024-04-26T04:52:59.558 INFO:teuthology.orchestra.run.smithi005.stdout:prometheus.a smithi111 *:9095 running (3m) 1s ago 9m 62.7M - 2.43.0 a07b618ecd1d 9e5fc4ef6c62 2024-04-26T04:52:59.570 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 31 tests from TestDeepCopy (54246 ms total) 2024-04-26T04:52:59.570 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:52:59.570 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 51 tests from TestMirroring 2024-04-26T04:52:59.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:52:59 smithi005 ceph-mon[91801]: pgmap v268: 105 pgs: 105 active+clean; 965 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 40 MiB/s wr, 638 op/s 2024-04-26T04:52:59.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:52:59 smithi005 ceph-mon[95809]: pgmap v268: 105 pgs: 105 active+clean; 965 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 40 MiB/s wr, 638 op/s 2024-04-26T04:53:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:52:59 smithi111 ceph-mon[86360]: pgmap v268: 105 pgs: 105 active+clean; 965 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 40 MiB/s wr, 638 op/s 2024-04-26T04:53:00.215 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:53:00.480 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:00.890 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[91801]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[91801]: osdmap e430: 8 total, 8 up, 8 in 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[95809]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[95809]: osdmap e430: 8 total, 8 up, 8 in 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:00.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:00 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:00.907 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:00 smithi111 ceph-mon[86360]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:00.907 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:00 smithi111 ceph-mon[86360]: osdmap e430: 8 total, 8 up, 8 in 2024-04-26T04:53:00.907 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:00 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:00.907 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:00 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:01.581 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-04-26T04:53:01.670 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[91801]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:53:01.670 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[91801]: osdmap e431: 8 total, 8 up, 8 in 2024-04-26T04:53:01.670 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:53:01.670 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:53:01.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[95809]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:53:01.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[95809]: osdmap e431: 8 total, 8 up, 8 in 2024-04-26T04:53:01.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:53:01.671 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:01 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:53:01.875 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:01 smithi111 ceph-mon[86360]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:53:01.875 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:01 smithi111 ceph-mon[86360]: osdmap e431: 8 total, 8 up, 8 in 2024-04-26T04:53:01.875 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:01 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:53:01.875 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:01 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:53:02.385 INFO:teuthology.orchestra.run.smithi005.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[95809]: osdmap e432: 8 total, 8 up, 8 in 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[91801]: osdmap e432: 8 total, 8 up, 8 in 2024-04-26T04:53:02.711 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:02.712 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:02.854 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:02 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:53:02.854 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:02 smithi111 ceph-mon[86360]: osdmap e432: 8 total, 8 up, 8 in 2024-04-26T04:53:02.854 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:02.854 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.039 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:53:03.306 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:03.430 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:53:03] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.43.0" 2024-04-26T04:53:03.696 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2115 ms) 2024-04-26T04:53:03.696 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-04-26T04:53:03.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:03.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: pgmap v273: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: osdmap e433: 8 total, 8 up, 8 in 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:53:03.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: pgmap v273: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:53:03.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: osdmap e433: 8 total, 8 up, 8 in 2024-04-26T04:53:03.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:03 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: pgmap v273: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: osdmap e433: 8 total, 8 up, 8 in 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:03 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.801 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (105 ms) 2024-04-26T04:53:03.801 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-04-26T04:53:03.878 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (77 ms) 2024-04-26T04:53:03.879 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-04-26T04:53:03.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (98 ms) 2024-04-26T04:53:03.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-04-26T04:53:04.066 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (90 ms) 2024-04-26T04:53:04.066 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-04-26T04:53:04.158 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (91 ms) 2024-04-26T04:53:04.158 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (108 ms) 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-04-26T04:53:04.265 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-04-26T04:53:04.332 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (67 ms) 2024-04-26T04:53:04.332 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-04-26T04:53:04.404 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (72 ms) 2024-04-26T04:53:04.404 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-04-26T04:53:04.713 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: pgmap v275: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:04.713 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: pgmap v276: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:04.713 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: osdmap e434: 8 total, 8 up, 8 in 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[95809]: Reconfiguring daemon mon.a on smithi005 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: pgmap v275: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: pgmap v276: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:04.714 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: osdmap e434: 8 total, 8 up, 8 in 2024-04-26T04:53:04.715 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:53:04.715 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:04.715 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:04.715 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:04.715 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:04 smithi005 ceph-mon[91801]: Reconfiguring daemon mon.a on smithi005 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: pgmap v275: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: pgmap v276: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: osdmap e434: 8 total, 8 up, 8 in 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:04 smithi111 ceph-mon[86360]: Reconfiguring daemon mon.a on smithi005 2024-04-26T04:53:05.657 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1253 ms) 2024-04-26T04:53:05.657 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-04-26T04:53:05.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:05 smithi005 ceph-mon[91801]: osdmap e435: 8 total, 8 up, 8 in 2024-04-26T04:53:05.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:05 smithi005 ceph-mon[95809]: osdmap e435: 8 total, 8 up, 8 in 2024-04-26T04:53:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:05 smithi111 ceph-mon[86360]: osdmap e435: 8 total, 8 up, 8 in 2024-04-26T04:53:06.895 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: pgmap v279: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:06.895 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: osdmap e436: 8 total, 8 up, 8 in 2024-04-26T04:53:06.895 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:06.895 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[95809]: Reconfiguring daemon mgr.y on smithi005 2024-04-26T04:53:06.896 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: pgmap v279: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: osdmap e436: 8 total, 8 up, 8 in 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:06.897 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:06 smithi005 ceph-mon[91801]: Reconfiguring daemon mgr.y on smithi005 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: pgmap v279: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: osdmap e436: 8 total, 8 up, 8 in 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:53:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:07.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:06 smithi111 ceph-mon[86360]: Reconfiguring daemon mgr.y on smithi005 2024-04-26T04:53:07.665 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2007 ms) 2024-04-26T04:53:07.665 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-04-26T04:53:07.762 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (97 ms) 2024-04-26T04:53:07.762 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-04-26T04:53:07.852 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (90 ms) 2024-04-26T04:53:07.852 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-04-26T04:53:07.946 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (95 ms) 2024-04-26T04:53:07.947 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-04-26T04:53:07.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:07 smithi005 ceph-mon[91801]: osdmap e437: 8 total, 8 up, 8 in 2024-04-26T04:53:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:07 smithi005 ceph-mon[95809]: osdmap e437: 8 total, 8 up, 8 in 2024-04-26T04:53:08.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:07 smithi111 ceph-mon[86360]: osdmap e437: 8 total, 8 up, 8 in 2024-04-26T04:53:08.022 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (75 ms) 2024-04-26T04:53:08.022 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-04-26T04:53:08.097 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (75 ms) 2024-04-26T04:53:08.098 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-04-26T04:53:08.187 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (90 ms) 2024-04-26T04:53:08.187 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-04-26T04:53:08.281 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (94 ms) 2024-04-26T04:53:08.281 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-04-26T04:53:08.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: pgmap v282: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 433 KiB/s rd, 58 KiB/s wr, 624 op/s 2024-04-26T04:53:08.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: osdmap e438: 8 total, 8 up, 8 in 2024-04-26T04:53:08.728 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: osdmap e439: 8 total, 8 up, 8 in 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[91801]: Reconfiguring daemon mon.c on smithi005 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: pgmap v282: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 433 KiB/s rd, 58 KiB/s wr, 624 op/s 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: osdmap e438: 8 total, 8 up, 8 in 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: osdmap e439: 8 total, 8 up, 8 in 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:08.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:08.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:53:08.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:08.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:08.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:08.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:08 smithi005 ceph-mon[95809]: Reconfiguring daemon mon.c on smithi005 2024-04-26T04:53:09.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:53:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:53:09.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: pgmap v282: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 433 KiB/s rd, 58 KiB/s wr, 624 op/s 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: osdmap e438: 8 total, 8 up, 8 in 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: osdmap e439: 8 total, 8 up, 8 in 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:09.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:09.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:09.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:08 smithi111 ceph-mon[86360]: Reconfiguring daemon mon.c on smithi005 2024-04-26T04:53:09.816 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1535 ms) 2024-04-26T04:53:09.816 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-04-26T04:53:09.927 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (111 ms) 2024-04-26T04:53:09.928 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-04-26T04:53:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:09 smithi111 ceph-mon[86360]: osdmap e440: 8 total, 8 up, 8 in 2024-04-26T04:53:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:09 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:09 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:09 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:53:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:09 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:10.028 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (101 ms) 2024-04-26T04:53:10.028 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-04-26T04:53:10.053 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[95809]: osdmap e440: 8 total, 8 up, 8 in 2024-04-26T04:53:10.053 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:10.053 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:10.053 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:53:10.054 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:10.054 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[91801]: osdmap e440: 8 total, 8 up, 8 in 2024-04-26T04:53:10.054 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:10.054 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:10.054 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:53:10.054 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:09 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:10.123 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (95 ms) 2024-04-26T04:53:10.124 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-04-26T04:53:10.281 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (158 ms) 2024-04-26T04:53:10.281 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-04-26T04:53:10.432 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (151 ms) 2024-04-26T04:53:10.432 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-04-26T04:53:10.560 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (128 ms) 2024-04-26T04:53:10.560 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-04-26T04:53:10.699 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (139 ms) 2024-04-26T04:53:10.699 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-04-26T04:53:10.851 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (152 ms) 2024-04-26T04:53:10.861 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-04-26T04:53:11.015 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (163 ms) 2024-04-26T04:53:11.015 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-04-26T04:53:11.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:10 smithi111 ceph-mon[86360]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:53:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:10 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.0 on smithi005 2024-04-26T04:53:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:10 smithi111 ceph-mon[86360]: pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 58 KiB/s wr, 628 op/s 2024-04-26T04:53:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:10 smithi111 ceph-mon[86360]: osdmap e441: 8 total, 8 up, 8 in 2024-04-26T04:53:11.092 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (77 ms) 2024-04-26T04:53:11.092 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-04-26T04:53:11.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[91801]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:53:11.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.0 on smithi005 2024-04-26T04:53:11.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[91801]: pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 58 KiB/s wr, 628 op/s 2024-04-26T04:53:11.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[91801]: osdmap e441: 8 total, 8 up, 8 in 2024-04-26T04:53:11.118 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[95809]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:53:11.118 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.0 on smithi005 2024-04-26T04:53:11.118 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[95809]: pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 58 KiB/s wr, 628 op/s 2024-04-26T04:53:11.118 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:10 smithi005 ceph-mon[95809]: osdmap e441: 8 total, 8 up, 8 in 2024-04-26T04:53:11.170 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (78 ms) 2024-04-26T04:53:11.170 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-04-26T04:53:11.238 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (68 ms) 2024-04-26T04:53:11.239 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-04-26T04:53:11.359 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (121 ms) 2024-04-26T04:53:11.359 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-04-26T04:53:11.475 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (116 ms) 2024-04-26T04:53:11.476 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-04-26T04:53:11.580 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (105 ms) 2024-04-26T04:53:11.580 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-04-26T04:53:11.690 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (110 ms) 2024-04-26T04:53:11.690 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-04-26T04:53:11.767 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.MirrorStatusList (77 ms) 2024-04-26T04:53:11.767 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-04-26T04:53:11.864 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.RemoveBootstrapped (96 ms) 2024-04-26T04:53:11.864 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-04-26T04:53:12.228 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.AioPromoteDemote (365 ms) 2024-04-26T04:53:12.229 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-04-26T04:53:12.456 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.456 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.456 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:53:12.456 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:53:12.456 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.456 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.1 on smithi005 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: pgmap v288: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.1 on smithi005 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: pgmap v288: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:12.457 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.1 on smithi005 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: pgmap v288: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:12 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:12.573 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.AioGetInfo (345 ms) 2024-04-26T04:53:12.573 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-04-26T04:53:12.806 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.AioGetStatus (233 ms) 2024-04-26T04:53:12.807 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.SiteName 2024-04-26T04:53:12.814 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.SiteName (8 ms) 2024-04-26T04:53:12.814 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.Bootstrap 2024-04-26T04:53:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:53:13] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.43.0" 2024-04-26T04:53:13.678 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.Bootstrap (864 ms) 2024-04-26T04:53:13.678 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.PeerDirection 2024-04-26T04:53:13.686 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.PeerDirection (8 ms) 2024-04-26T04:53:13.686 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.Snapshot 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:13.864 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.2 on smithi005 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]: dispatch 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]: dispatch 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]': finished 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' 2024-04-26T04:53:13.865 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.2 on smithi005 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-04-26T04:53:13.866 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-04-26T04:53:13.867 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-04-26T04:53:13.867 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:13.867 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]: dispatch 2024-04-26T04:53:13.867 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]: dispatch 2024-04-26T04:53:13.867 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]': finished 2024-04-26T04:53:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' 2024-04-26T04:53:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.2 on smithi005 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-04-26T04:53:14.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-04-26T04:53:14.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]: dispatch 2024-04-26T04:53:14.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]: dispatch 2024-04-26T04:53:14.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/62b71298-a33b-4a7f-8fd5-baa9243c6cd4"}]': finished 2024-04-26T04:53:15.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[95809]: pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 184 KiB/s wr, 1.74k op/s 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[95809]: osdmap e442: 8 total, 8 up, 8 in 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[91801]: pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 184 KiB/s wr, 1.74k op/s 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[91801]: osdmap e442: 8 total, 8 up, 8 in 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:53:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:15 smithi111 ceph-mon[86360]: pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 184 KiB/s wr, 1.74k op/s 2024-04-26T04:53:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:15 smithi111 ceph-mon[86360]: osdmap e442: 8 total, 8 up, 8 in 2024-04-26T04:53:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:15.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:53:15.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:16 smithi005 ceph-mon[95809]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:53:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:16 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.3 on smithi005 2024-04-26T04:53:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:16 smithi005 ceph-mon[95809]: osdmap e443: 8 total, 8 up, 8 in 2024-04-26T04:53:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:16 smithi005 ceph-mon[91801]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:53:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:16 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.3 on smithi005 2024-04-26T04:53:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:16 smithi005 ceph-mon[91801]: osdmap e443: 8 total, 8 up, 8 in 2024-04-26T04:53:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:16 smithi111 ceph-mon[86360]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:53:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:16 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.3 on smithi005 2024-04-26T04:53:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:16 smithi111 ceph-mon[86360]: osdmap e443: 8 total, 8 up, 8 in 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 161 KiB/s wr, 1.52k op/s 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: osdmap e444: 8 total, 8 up, 8 in 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 161 KiB/s wr, 1.52k op/s 2024-04-26T04:53:17.374 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:17.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:17.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:17.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: osdmap e444: 8 total, 8 up, 8 in 2024-04-26T04:53:17.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:17.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:17.375 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 161 KiB/s wr, 1.52k op/s 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: osdmap e444: 8 total, 8 up, 8 in 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi005.zdnlsb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:18 smithi005 ceph-mon[91801]: Reconfiguring mds.foo.smithi005.zdnlsb (monmap changed)... 2024-04-26T04:53:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:18 smithi005 ceph-mon[91801]: Reconfiguring daemon mds.foo.smithi005.zdnlsb on smithi005 2024-04-26T04:53:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:18 smithi005 ceph-mon[91801]: osdmap e445: 8 total, 8 up, 8 in 2024-04-26T04:53:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:18 smithi005 ceph-mon[95809]: Reconfiguring mds.foo.smithi005.zdnlsb (monmap changed)... 2024-04-26T04:53:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:18 smithi005 ceph-mon[95809]: Reconfiguring daemon mds.foo.smithi005.zdnlsb on smithi005 2024-04-26T04:53:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:18 smithi005 ceph-mon[95809]: osdmap e445: 8 total, 8 up, 8 in 2024-04-26T04:53:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:18 smithi111 ceph-mon[86360]: Reconfiguring mds.foo.smithi005.zdnlsb (monmap changed)... 2024-04-26T04:53:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:18 smithi111 ceph-mon[86360]: Reconfiguring daemon mds.foo.smithi005.zdnlsb on smithi005 2024-04-26T04:53:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:18 smithi111 ceph-mon[86360]: osdmap e445: 8 total, 8 up, 8 in 2024-04-26T04:53:19.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:53:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:53:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:53:19.399 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 235 KiB/s wr, 2.23k op/s 2024-04-26T04:53:19.400 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:19.400 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:19.400 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:19.400 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:19.400 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:19.400 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:19 smithi111 ceph-mon[86360]: osdmap e446: 8 total, 8 up, 8 in 2024-04-26T04:53:19.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 235 KiB/s wr, 2.23k op/s 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[91801]: osdmap e446: 8 total, 8 up, 8 in 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 235 KiB/s wr, 2.23k op/s 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:53:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:53:19.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:19.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:19 smithi005 ceph-mon[95809]: osdmap e446: 8 total, 8 up, 8 in 2024-04-26T04:53:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:20 smithi005 ceph-mon[91801]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:53:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:20 smithi005 ceph-mon[91801]: Reconfiguring daemon mon.b on smithi111 2024-04-26T04:53:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:20 smithi005 ceph-mon[91801]: osdmap e447: 8 total, 8 up, 8 in 2024-04-26T04:53:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:20 smithi005 ceph-mon[95809]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:53:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:20 smithi005 ceph-mon[95809]: Reconfiguring daemon mon.b on smithi111 2024-04-26T04:53:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:20 smithi005 ceph-mon[95809]: osdmap e447: 8 total, 8 up, 8 in 2024-04-26T04:53:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:20 smithi111 ceph-mon[86360]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:53:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:20 smithi111 ceph-mon[86360]: Reconfiguring daemon mon.b on smithi111 2024-04-26T04:53:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:20 smithi111 ceph-mon[86360]: osdmap e447: 8 total, 8 up, 8 in 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 545 KiB/s rd, 78 KiB/s wr, 753 op/s 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:21.410 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:21 smithi111 ceph-mon[86360]: osdmap e448: 8 total, 8 up, 8 in 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 545 KiB/s rd, 78 KiB/s wr, 753 op/s 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[91801]: osdmap e448: 8 total, 8 up, 8 in 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 545 KiB/s rd, 78 KiB/s wr, 753 op/s 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:53:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:53:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:21.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:21 smithi005 ceph-mon[95809]: osdmap e448: 8 total, 8 up, 8 in 2024-04-26T04:53:22.382 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:53:22.383 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: Reconfiguring daemon mgr.x on smithi111 2024-04-26T04:53:22.383 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: osdmap e449: 8 total, 8 up, 8 in 2024-04-26T04:53:22.383 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:22.383 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:22.383 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:53:22.383 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:22 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:22.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: Reconfiguring daemon mgr.x on smithi111 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: osdmap e449: 8 total, 8 up, 8 in 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:53:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: Reconfiguring daemon mgr.x on smithi111 2024-04-26T04:53:22.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: osdmap e449: 8 total, 8 up, 8 in 2024-04-26T04:53:22.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:22.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:22.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:53:22.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:22 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:53:23] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[91801]: pgmap v300: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[91801]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.4 on smithi111 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[91801]: osdmap e450: 8 total, 8 up, 8 in 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[95809]: pgmap v300: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[95809]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:53:23.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.4 on smithi111 2024-04-26T04:53:23.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:23 smithi005 ceph-mon[95809]: osdmap e450: 8 total, 8 up, 8 in 2024-04-26T04:53:23.422 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:23 smithi111 ceph-mon[86360]: pgmap v300: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:23.423 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:23 smithi111 ceph-mon[86360]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:53:23.423 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:23 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.4 on smithi111 2024-04-26T04:53:23.423 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:23 smithi111 ceph-mon[86360]: osdmap e450: 8 total, 8 up, 8 in 2024-04-26T04:53:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:53:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:53:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.5 on smithi111 2024-04-26T04:53:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: pgmap v303: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-04-26T04:53:24.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:24 smithi111 ceph-mon[86360]: osdmap e451: 8 total, 8 up, 8 in 2024-04-26T04:53:24.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.5 on smithi111 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: pgmap v303: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[91801]: osdmap e451: 8 total, 8 up, 8 in 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:24.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:24.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:53:24.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:53:24.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:24.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.5 on smithi111 2024-04-26T04:53:24.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: pgmap v303: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-04-26T04:53:24.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:24 smithi005 ceph-mon[95809]: osdmap e451: 8 total, 8 up, 8 in 2024-04-26T04:53:25.905 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: osdmap e452: 8 total, 8 up, 8 in 2024-04-26T04:53:25.905 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:25.906 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:25.906 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:53:25.906 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:53:25.906 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:25.906 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:25 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.6 on smithi111 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: osdmap e452: 8 total, 8 up, 8 in 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.6 on smithi111 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: osdmap e452: 8 total, 8 up, 8 in 2024-04-26T04:53:25.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:25.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:25.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:53:25.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:53:25.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:25.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:25 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.6 on smithi111 2024-04-26T04:53:26.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:26 smithi005 ceph-mon[91801]: pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-04-26T04:53:26.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:26 smithi005 ceph-mon[91801]: osdmap e453: 8 total, 8 up, 8 in 2024-04-26T04:53:26.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:26 smithi005 ceph-mon[95809]: pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-04-26T04:53:26.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:26 smithi005 ceph-mon[95809]: osdmap e453: 8 total, 8 up, 8 in 2024-04-26T04:53:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:26 smithi111 ceph-mon[86360]: pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-04-26T04:53:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:26 smithi111 ceph-mon[86360]: osdmap e453: 8 total, 8 up, 8 in 2024-04-26T04:53:27.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: osdmap e454: 8 total, 8 up, 8 in 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: Reconfiguring daemon osd.7 on smithi111 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[91801]: osdmap e455: 8 total, 8 up, 8 in 2024-04-26T04:53:27.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: osdmap e454: 8 total, 8 up, 8 in 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: Reconfiguring daemon osd.7 on smithi111 2024-04-26T04:53:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:27 smithi005 ceph-mon[95809]: osdmap e455: 8 total, 8 up, 8 in 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: osdmap e454: 8 total, 8 up, 8 in 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: Reconfiguring daemon osd.7 on smithi111 2024-04-26T04:53:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:27 smithi111 ceph-mon[86360]: osdmap e455: 8 total, 8 up, 8 in 2024-04-26T04:53:29.018 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 04:53:28 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T04:53:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi111.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.5\", device_class=\"ssd\", hostname=\"smithi005\", instance=\"smithi005:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.5\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:53:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:28 smithi111 ceph-mon[86360]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-04-26T04:53:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:28 smithi111 ceph-mon[86360]: osdmap e456: 8 total, 8 up, 8 in 2024-04-26T04:53:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:28 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:28 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[91801]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-04-26T04:53:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[91801]: osdmap e456: 8 total, 8 up, 8 in 2024-04-26T04:53:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[95809]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-04-26T04:53:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[95809]: osdmap e456: 8 total, 8 up, 8 in 2024-04-26T04:53:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:28 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:29.743 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.Snapshot (16058 ms) 2024-04-26T04:53:29.744 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-04-26T04:53:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: Reconfiguring mds.foo.smithi111.furhfd (monmap changed)... 2024-04-26T04:53:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: Reconfiguring daemon mds.foo.smithi111.furhfd on smithi111 2024-04-26T04:53:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: osdmap e457: 8 total, 8 up, 8 in 2024-04-26T04:53:30.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]: dispatch 2024-04-26T04:53:30.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]: dispatch 2024-04-26T04:53:30.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:29 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]': finished 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: Reconfiguring mds.foo.smithi111.furhfd (monmap changed)... 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: Reconfiguring daemon mds.foo.smithi111.furhfd on smithi111 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: osdmap e457: 8 total, 8 up, 8 in 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]: dispatch 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]: dispatch 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]': finished 2024-04-26T04:53:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: Reconfiguring mds.foo.smithi111.furhfd (monmap changed)... 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi111.furhfd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: Reconfiguring daemon mds.foo.smithi111.furhfd on smithi111 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: osdmap e457: 8 total, 8 up, 8 in 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]: dispatch 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]: dispatch 2024-04-26T04:53:30.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:29 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/707e2b27-3995-4330-a8ca-155a6f3fb4a0"}]': finished 2024-04-26T04:53:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:30 smithi111 ceph-mon[86360]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-04-26T04:53:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:30 smithi111 ceph-mon[86360]: osdmap e458: 8 total, 8 up, 8 in 2024-04-26T04:53:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:30 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:30 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[91801]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[91801]: osdmap e458: 8 total, 8 up, 8 in 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[95809]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[95809]: osdmap e458: 8 total, 8 up, 8 in 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:30 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:32.203 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: pgmap v315: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:32.204 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:31 smithi111 ceph-mon[86360]: osdmap e459: 8 total, 8 up, 8 in 2024-04-26T04:53:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: pgmap v315: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:32.233 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[91801]: osdmap e459: 8 total, 8 up, 8 in 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: pgmap v315: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:32.234 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:31 smithi005 ceph-mon[95809]: osdmap e459: 8 total, 8 up, 8 in 2024-04-26T04:53:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:53:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:53:33] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.43.0" 2024-04-26T04:53:33.836 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (4092 ms) 2024-04-26T04:53:33.836 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-04-26T04:53:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:33 smithi111 ceph-mon[86360]: osdmap e460: 8 total, 8 up, 8 in 2024-04-26T04:53:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:33 smithi111 ceph-mon[86360]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 12 KiB/s wr, 149 op/s 2024-04-26T04:53:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:33 smithi005 ceph-mon[91801]: osdmap e460: 8 total, 8 up, 8 in 2024-04-26T04:53:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:33 smithi005 ceph-mon[91801]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 12 KiB/s wr, 149 op/s 2024-04-26T04:53:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:33 smithi005 ceph-mon[95809]: osdmap e460: 8 total, 8 up, 8 in 2024-04-26T04:53:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:33 smithi005 ceph-mon[95809]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 12 KiB/s wr, 149 op/s 2024-04-26T04:53:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:34 smithi111 ceph-mon[86360]: osdmap e461: 8 total, 8 up, 8 in 2024-04-26T04:53:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:34 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]: dispatch 2024-04-26T04:53:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:34 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]: dispatch 2024-04-26T04:53:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:34 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]': finished 2024-04-26T04:53:35.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[91801]: osdmap e461: 8 total, 8 up, 8 in 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]: dispatch 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]: dispatch 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]': finished 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[95809]: osdmap e461: 8 total, 8 up, 8 in 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]: dispatch 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]: dispatch 2024-04-26T04:53:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:34 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/941c93f5-99c9-40c1-9347-413c4b2ce8dc"}]': finished 2024-04-26T04:53:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:35 smithi005 ceph-mon[91801]: pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 9.5 KiB/s wr, 118 op/s 2024-04-26T04:53:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:35 smithi005 ceph-mon[91801]: osdmap e462: 8 total, 8 up, 8 in 2024-04-26T04:53:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:35 smithi005 ceph-mon[95809]: pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 9.5 KiB/s wr, 118 op/s 2024-04-26T04:53:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:35 smithi005 ceph-mon[95809]: osdmap e462: 8 total, 8 up, 8 in 2024-04-26T04:53:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:35 smithi111 ceph-mon[86360]: pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 9.5 KiB/s wr, 118 op/s 2024-04-26T04:53:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:35 smithi111 ceph-mon[86360]: osdmap e462: 8 total, 8 up, 8 in 2024-04-26T04:53:37.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:36 smithi005 ceph-mon[91801]: osdmap e463: 8 total, 8 up, 8 in 2024-04-26T04:53:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:36 smithi005 ceph-mon[95809]: osdmap e463: 8 total, 8 up, 8 in 2024-04-26T04:53:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:36 smithi111 ceph-mon[86360]: osdmap e463: 8 total, 8 up, 8 in 2024-04-26T04:53:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:37 smithi005 ceph-mon[91801]: pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:37 smithi005 ceph-mon[91801]: osdmap e464: 8 total, 8 up, 8 in 2024-04-26T04:53:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:37 smithi005 ceph-mon[91801]: osdmap e465: 8 total, 8 up, 8 in 2024-04-26T04:53:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:37 smithi005 ceph-mon[95809]: pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:37 smithi005 ceph-mon[95809]: osdmap e464: 8 total, 8 up, 8 in 2024-04-26T04:53:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:37 smithi005 ceph-mon[95809]: osdmap e465: 8 total, 8 up, 8 in 2024-04-26T04:53:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:37 smithi111 ceph-mon[86360]: pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:37 smithi111 ceph-mon[86360]: osdmap e464: 8 total, 8 up, 8 in 2024-04-26T04:53:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:37 smithi111 ceph-mon[86360]: osdmap e465: 8 total, 8 up, 8 in 2024-04-26T04:53:39.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[91801]: osdmap e466: 8 total, 8 up, 8 in 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]: dispatch 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]: dispatch 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]': finished 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[91801]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 151 op/s 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[95809]: osdmap e466: 8 total, 8 up, 8 in 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]: dispatch 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]: dispatch 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]': finished 2024-04-26T04:53:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:39 smithi005 ceph-mon[95809]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 151 op/s 2024-04-26T04:53:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:39 smithi111 ceph-mon[86360]: osdmap e466: 8 total, 8 up, 8 in 2024-04-26T04:53:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:39 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]: dispatch 2024-04-26T04:53:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:39 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]: dispatch 2024-04-26T04:53:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:39 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/951be024-ace5-40bf-aa40-9a158899af9b"}]': finished 2024-04-26T04:53:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:39 smithi111 ceph-mon[86360]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 151 op/s 2024-04-26T04:53:40.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:40 smithi005 ceph-mon[91801]: osdmap e467: 8 total, 8 up, 8 in 2024-04-26T04:53:40.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:40 smithi005 ceph-mon[95809]: osdmap e467: 8 total, 8 up, 8 in 2024-04-26T04:53:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:40 smithi111 ceph-mon[86360]: osdmap e467: 8 total, 8 up, 8 in 2024-04-26T04:53:41.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:41 smithi005 ceph-mon[91801]: osdmap e468: 8 total, 8 up, 8 in 2024-04-26T04:53:41.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:41 smithi005 ceph-mon[91801]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 152 op/s 2024-04-26T04:53:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:41 smithi005 ceph-mon[95809]: osdmap e468: 8 total, 8 up, 8 in 2024-04-26T04:53:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:41 smithi005 ceph-mon[95809]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 152 op/s 2024-04-26T04:53:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:41 smithi111 ceph-mon[86360]: osdmap e468: 8 total, 8 up, 8 in 2024-04-26T04:53:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:41 smithi111 ceph-mon[86360]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 152 op/s 2024-04-26T04:53:42.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:42 smithi005 ceph-mon[91801]: osdmap e469: 8 total, 8 up, 8 in 2024-04-26T04:53:42.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:42 smithi005 ceph-mon[95809]: osdmap e469: 8 total, 8 up, 8 in 2024-04-26T04:53:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:42 smithi111 ceph-mon[86360]: osdmap e469: 8 total, 8 up, 8 in 2024-04-26T04:53:43.481 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:53:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:53:43] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.43.0" 2024-04-26T04:53:43.759 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9922 ms) 2024-04-26T04:53:43.759 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-04-26T04:53:43.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:43 smithi005 ceph-mon[91801]: osdmap e470: 8 total, 8 up, 8 in 2024-04-26T04:53:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:43 smithi005 ceph-mon[91801]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 9.7 KiB/s wr, 221 op/s 2024-04-26T04:53:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:43 smithi005 ceph-mon[95809]: osdmap e470: 8 total, 8 up, 8 in 2024-04-26T04:53:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:43 smithi005 ceph-mon[95809]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 9.7 KiB/s wr, 221 op/s 2024-04-26T04:53:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:43 smithi111 ceph-mon[86360]: osdmap e470: 8 total, 8 up, 8 in 2024-04-26T04:53:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:43 smithi111 ceph-mon[86360]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 9.7 KiB/s wr, 221 op/s 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: osdmap e471: 8 total, 8 up, 8 in 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]: dispatch 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]: dispatch 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]': finished 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]: dispatch 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]: dispatch 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]': finished 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: osdmap e471: 8 total, 8 up, 8 in 2024-04-26T04:53:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]: dispatch 2024-04-26T04:53:44.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]: dispatch 2024-04-26T04:53:44.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]': finished 2024-04-26T04:53:44.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]: dispatch 2024-04-26T04:53:44.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]: dispatch 2024-04-26T04:53:44.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:44 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]': finished 2024-04-26T04:53:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: osdmap e471: 8 total, 8 up, 8 in 2024-04-26T04:53:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]: dispatch 2024-04-26T04:53:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]: dispatch 2024-04-26T04:53:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/cd8f1a14-1cb0-4bcd-b0a3-a6979a0c7322"}]': finished 2024-04-26T04:53:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]: dispatch 2024-04-26T04:53:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]: dispatch 2024-04-26T04:53:45.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:44 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ae05b4f0-0a55-4b9d-857f-0de8bdb421f5"}]': finished 2024-04-26T04:53:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:45 smithi111 ceph-mon[86360]: osdmap e472: 8 total, 8 up, 8 in 2024-04-26T04:53:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:45 smithi111 ceph-mon[86360]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 9.7 KiB/s wr, 221 op/s 2024-04-26T04:53:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:45 smithi005 ceph-mon[91801]: osdmap e472: 8 total, 8 up, 8 in 2024-04-26T04:53:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:45 smithi005 ceph-mon[91801]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 9.7 KiB/s wr, 221 op/s 2024-04-26T04:53:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:45 smithi005 ceph-mon[95809]: osdmap e472: 8 total, 8 up, 8 in 2024-04-26T04:53:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:45 smithi005 ceph-mon[95809]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 9.7 KiB/s wr, 221 op/s 2024-04-26T04:53:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:46 smithi111 ceph-mon[86360]: osdmap e473: 8 total, 8 up, 8 in 2024-04-26T04:53:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:46 smithi005 ceph-mon[91801]: osdmap e473: 8 total, 8 up, 8 in 2024-04-26T04:53:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:46 smithi005 ceph-mon[95809]: osdmap e473: 8 total, 8 up, 8 in 2024-04-26T04:53:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:47.692 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.SnapshotImageState (3933 ms) 2024-04-26T04:53:47.692 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-04-26T04:53:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:47 smithi005 ceph-mon[91801]: osdmap e474: 8 total, 8 up, 8 in 2024-04-26T04:53:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:47 smithi005 ceph-mon[91801]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:47 smithi005 ceph-mon[91801]: osdmap e475: 8 total, 8 up, 8 in 2024-04-26T04:53:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:47 smithi005 ceph-mon[95809]: osdmap e474: 8 total, 8 up, 8 in 2024-04-26T04:53:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:47 smithi005 ceph-mon[95809]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:47 smithi005 ceph-mon[95809]: osdmap e475: 8 total, 8 up, 8 in 2024-04-26T04:53:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:47 smithi111 ceph-mon[86360]: osdmap e474: 8 total, 8 up, 8 in 2024-04-26T04:53:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:47 smithi111 ceph-mon[86360]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:47 smithi111 ceph-mon[86360]: osdmap e475: 8 total, 8 up, 8 in 2024-04-26T04:53:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:49 smithi005 ceph-mon[91801]: osdmap e476: 8 total, 8 up, 8 in 2024-04-26T04:53:49.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:49 smithi005 ceph-mon[91801]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 24 KiB/s wr, 224 op/s 2024-04-26T04:53:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:49 smithi005 ceph-mon[95809]: osdmap e476: 8 total, 8 up, 8 in 2024-04-26T04:53:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:49 smithi005 ceph-mon[95809]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 24 KiB/s wr, 224 op/s 2024-04-26T04:53:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:49 smithi111 ceph-mon[86360]: osdmap e476: 8 total, 8 up, 8 in 2024-04-26T04:53:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:49 smithi111 ceph-mon[86360]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 24 KiB/s wr, 224 op/s 2024-04-26T04:53:50.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:50 smithi005 ceph-mon[91801]: osdmap e477: 8 total, 8 up, 8 in 2024-04-26T04:53:50.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:50 smithi005 ceph-mon[95809]: osdmap e477: 8 total, 8 up, 8 in 2024-04-26T04:53:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:50 smithi111 ceph-mon[86360]: osdmap e477: 8 total, 8 up, 8 in 2024-04-26T04:53:51.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:51 smithi005 ceph-mon[95809]: osdmap e478: 8 total, 8 up, 8 in 2024-04-26T04:53:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:51 smithi005 ceph-mon[95809]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 24 KiB/s wr, 224 op/s 2024-04-26T04:53:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:51 smithi005 ceph-mon[91801]: osdmap e478: 8 total, 8 up, 8 in 2024-04-26T04:53:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:51 smithi005 ceph-mon[91801]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 24 KiB/s wr, 224 op/s 2024-04-26T04:53:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:51 smithi111 ceph-mon[86360]: osdmap e478: 8 total, 8 up, 8 in 2024-04-26T04:53:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:51 smithi111 ceph-mon[86360]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 24 KiB/s wr, 224 op/s 2024-04-26T04:53:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:52 smithi005 ceph-mon[91801]: osdmap e479: 8 total, 8 up, 8 in 2024-04-26T04:53:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:52 smithi005 ceph-mon[95809]: osdmap e479: 8 total, 8 up, 8 in 2024-04-26T04:53:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:52 smithi111 ceph-mon[86360]: osdmap e479: 8 total, 8 up, 8 in 2024-04-26T04:53:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:53:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:53:53] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2024-04-26T04:53:53.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:53 smithi005 ceph-mon[91801]: osdmap e480: 8 total, 8 up, 8 in 2024-04-26T04:53:53.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:53 smithi005 ceph-mon[91801]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.5 KiB/s wr, 136 op/s 2024-04-26T04:53:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:53 smithi005 ceph-mon[95809]: osdmap e480: 8 total, 8 up, 8 in 2024-04-26T04:53:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:53 smithi005 ceph-mon[95809]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.5 KiB/s wr, 136 op/s 2024-04-26T04:53:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:53 smithi111 ceph-mon[86360]: osdmap e480: 8 total, 8 up, 8 in 2024-04-26T04:53:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:53 smithi111 ceph-mon[86360]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.5 KiB/s wr, 136 op/s 2024-04-26T04:53:54.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:54 smithi005 ceph-mon[91801]: osdmap e481: 8 total, 8 up, 8 in 2024-04-26T04:53:54.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:54 smithi005 ceph-mon[95809]: osdmap e481: 8 total, 8 up, 8 in 2024-04-26T04:53:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:54 smithi111 ceph-mon[86360]: osdmap e481: 8 total, 8 up, 8 in 2024-04-26T04:53:55.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:55 smithi005 ceph-mon[91801]: osdmap e482: 8 total, 8 up, 8 in 2024-04-26T04:53:55.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:55 smithi005 ceph-mon[91801]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.5 KiB/s wr, 136 op/s 2024-04-26T04:53:55.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:55 smithi005 ceph-mon[95809]: osdmap e482: 8 total, 8 up, 8 in 2024-04-26T04:53:55.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:55 smithi005 ceph-mon[95809]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.5 KiB/s wr, 136 op/s 2024-04-26T04:53:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:55 smithi111 ceph-mon[86360]: osdmap e482: 8 total, 8 up, 8 in 2024-04-26T04:53:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:55 smithi111 ceph-mon[86360]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.5 KiB/s wr, 136 op/s 2024-04-26T04:53:56.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:56 smithi005 ceph-mon[91801]: osdmap e483: 8 total, 8 up, 8 in 2024-04-26T04:53:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:56 smithi005 ceph-mon[95809]: osdmap e483: 8 total, 8 up, 8 in 2024-04-26T04:53:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:56 smithi111 ceph-mon[86360]: osdmap e483: 8 total, 8 up, 8 in 2024-04-26T04:53:57.706 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (10014 ms) 2024-04-26T04:53:57.707 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-04-26T04:53:57.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:57 smithi005 ceph-mon[95809]: osdmap e484: 8 total, 8 up, 8 in 2024-04-26T04:53:57.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:57 smithi005 ceph-mon[95809]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:57.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:57 smithi005 ceph-mon[95809]: osdmap e485: 8 total, 8 up, 8 in 2024-04-26T04:53:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:57 smithi005 ceph-mon[91801]: osdmap e484: 8 total, 8 up, 8 in 2024-04-26T04:53:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:57 smithi005 ceph-mon[91801]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:57 smithi005 ceph-mon[91801]: osdmap e485: 8 total, 8 up, 8 in 2024-04-26T04:53:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:57 smithi111 ceph-mon[86360]: osdmap e484: 8 total, 8 up, 8 in 2024-04-26T04:53:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:57 smithi111 ceph-mon[86360]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:57 smithi111 ceph-mon[86360]: osdmap e485: 8 total, 8 up, 8 in 2024-04-26T04:53:58.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:58 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]: dispatch 2024-04-26T04:53:58.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:58 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]: dispatch 2024-04-26T04:53:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:58 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]': finished 2024-04-26T04:53:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:58 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]: dispatch 2024-04-26T04:53:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:58 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]: dispatch 2024-04-26T04:53:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:58 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]': finished 2024-04-26T04:53:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:58 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]: dispatch 2024-04-26T04:53:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:58 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]: dispatch 2024-04-26T04:53:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:58 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e9db9720-3396-4ce9-96e8-d7c371d8cc83"}]': finished 2024-04-26T04:53:59.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:59 smithi005 ceph-mon[91801]: osdmap e486: 8 total, 8 up, 8 in 2024-04-26T04:53:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:53:59 smithi005 ceph-mon[91801]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 6.7 KiB/s wr, 82 op/s 2024-04-26T04:53:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:59 smithi005 ceph-mon[95809]: osdmap e486: 8 total, 8 up, 8 in 2024-04-26T04:53:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:53:59 smithi005 ceph-mon[95809]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 6.7 KiB/s wr, 82 op/s 2024-04-26T04:54:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:59 smithi111 ceph-mon[86360]: osdmap e486: 8 total, 8 up, 8 in 2024-04-26T04:54:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:53:59 smithi111 ceph-mon[86360]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 6.7 KiB/s wr, 82 op/s 2024-04-26T04:54:00.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:00 smithi005 ceph-mon[91801]: osdmap e487: 8 total, 8 up, 8 in 2024-04-26T04:54:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:00 smithi005 ceph-mon[95809]: osdmap e487: 8 total, 8 up, 8 in 2024-04-26T04:54:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:00 smithi111 ceph-mon[86360]: osdmap e487: 8 total, 8 up, 8 in 2024-04-26T04:54:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:01 smithi005 ceph-mon[91801]: osdmap e488: 8 total, 8 up, 8 in 2024-04-26T04:54:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:01 smithi005 ceph-mon[91801]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 6.7 KiB/s wr, 82 op/s 2024-04-26T04:54:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:01 smithi005 ceph-mon[95809]: osdmap e488: 8 total, 8 up, 8 in 2024-04-26T04:54:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:01 smithi005 ceph-mon[95809]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 6.7 KiB/s wr, 82 op/s 2024-04-26T04:54:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:01 smithi111 ceph-mon[86360]: osdmap e488: 8 total, 8 up, 8 in 2024-04-26T04:54:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:01 smithi111 ceph-mon[86360]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 6.7 KiB/s wr, 82 op/s 2024-04-26T04:54:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:02.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:02 smithi005 ceph-mon[95809]: osdmap e489: 8 total, 8 up, 8 in 2024-04-26T04:54:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:02 smithi005 ceph-mon[91801]: osdmap e489: 8 total, 8 up, 8 in 2024-04-26T04:54:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:02 smithi111 ceph-mon[86360]: osdmap e489: 8 total, 8 up, 8 in 2024-04-26T04:54:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:54:03] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2024-04-26T04:54:03.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:03 smithi005 ceph-mon[91801]: osdmap e490: 8 total, 8 up, 8 in 2024-04-26T04:54:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:03 smithi005 ceph-mon[91801]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 130 op/s 2024-04-26T04:54:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:03 smithi005 ceph-mon[95809]: osdmap e490: 8 total, 8 up, 8 in 2024-04-26T04:54:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:03 smithi005 ceph-mon[95809]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 130 op/s 2024-04-26T04:54:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:03 smithi111 ceph-mon[86360]: osdmap e490: 8 total, 8 up, 8 in 2024-04-26T04:54:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:03 smithi111 ceph-mon[86360]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 130 op/s 2024-04-26T04:54:04.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:04 smithi005 ceph-mon[91801]: osdmap e491: 8 total, 8 up, 8 in 2024-04-26T04:54:04.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:04 smithi005 ceph-mon[95809]: osdmap e491: 8 total, 8 up, 8 in 2024-04-26T04:54:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:04 smithi111 ceph-mon[86360]: osdmap e491: 8 total, 8 up, 8 in 2024-04-26T04:54:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:05 smithi111 ceph-mon[86360]: osdmap e492: 8 total, 8 up, 8 in 2024-04-26T04:54:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:05 smithi111 ceph-mon[86360]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 130 op/s 2024-04-26T04:54:06.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:05 smithi005 ceph-mon[95809]: osdmap e492: 8 total, 8 up, 8 in 2024-04-26T04:54:06.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:05 smithi005 ceph-mon[95809]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 130 op/s 2024-04-26T04:54:06.198 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:05 smithi005 ceph-mon[91801]: osdmap e492: 8 total, 8 up, 8 in 2024-04-26T04:54:06.198 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:05 smithi005 ceph-mon[91801]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 14 KiB/s wr, 130 op/s 2024-04-26T04:54:06.229 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:54:06.502 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:06.893 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:06 smithi005 ceph-mon[91801]: osdmap e493: 8 total, 8 up, 8 in 2024-04-26T04:54:06.893 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:06 smithi005 ceph-mon[95809]: osdmap e493: 8 total, 8 up, 8 in 2024-04-26T04:54:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:06 smithi111 ceph-mon[86360]: osdmap e493: 8 total, 8 up, 8 in 2024-04-26T04:54:07.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:07 smithi005 ceph-mon[91801]: osdmap e494: 8 total, 8 up, 8 in 2024-04-26T04:54:07.937 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:07 smithi005 ceph-mon[91801]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:07.937 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:07 smithi005 ceph-mon[91801]: osdmap e495: 8 total, 8 up, 8 in 2024-04-26T04:54:07.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:07 smithi005 ceph-mon[95809]: osdmap e494: 8 total, 8 up, 8 in 2024-04-26T04:54:07.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:07 smithi005 ceph-mon[95809]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:07.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:07 smithi005 ceph-mon[95809]: osdmap e495: 8 total, 8 up, 8 in 2024-04-26T04:54:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:07 smithi111 ceph-mon[86360]: osdmap e494: 8 total, 8 up, 8 in 2024-04-26T04:54:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:07 smithi111 ceph-mon[86360]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:07 smithi111 ceph-mon[86360]: osdmap e495: 8 total, 8 up, 8 in 2024-04-26T04:54:08.414 INFO:teuthology.orchestra.run.smithi005.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:09.006 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for half of osds to upgrade' 2024-04-26T04:54:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:09 smithi111 ceph-mon[86360]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:09 smithi111 ceph-mon[86360]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:09 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:09 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:09 smithi111 ceph-mon[86360]: osdmap e496: 8 total, 8 up, 8 in 2024-04-26T04:54:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:09 smithi111 ceph-mon[86360]: pgmap v373: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-04-26T04:54:09.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[91801]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[91801]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[91801]: osdmap e496: 8 total, 8 up, 8 in 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[91801]: pgmap v373: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[95809]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[95809]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:09.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[95809]: osdmap e496: 8 total, 8 up, 8 in 2024-04-26T04:54:09.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:09 smithi005 ceph-mon[95809]: pgmap v373: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-04-26T04:54:09.927 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:10.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:10 smithi005 ceph-mon[91801]: osdmap e497: 8 total, 8 up, 8 in 2024-04-26T04:54:10.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:10 smithi005 ceph-mon[95809]: osdmap e497: 8 total, 8 up, 8 in 2024-04-26T04:54:11.005 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:10 smithi111 ceph-mon[86360]: osdmap e497: 8 total, 8 up, 8 in 2024-04-26T04:54:11.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:11 smithi005 ceph-mon[95809]: osdmap e498: 8 total, 8 up, 8 in 2024-04-26T04:54:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:11 smithi005 ceph-mon[95809]: pgmap v376: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-04-26T04:54:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:11 smithi005 ceph-mon[91801]: osdmap e498: 8 total, 8 up, 8 in 2024-04-26T04:54:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:11 smithi005 ceph-mon[91801]: pgmap v376: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-04-26T04:54:12.009 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:11 smithi111 ceph-mon[86360]: osdmap e498: 8 total, 8 up, 8 in 2024-04-26T04:54:12.009 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:11 smithi111 ceph-mon[86360]: pgmap v376: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-04-26T04:54:12.284 INFO:teuthology.orchestra.run.smithi005.stdout:wait for half of osds to upgrade 2024-04-26T04:54:12.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:12 smithi005 ceph-mon[95809]: osdmap e499: 8 total, 8 up, 8 in 2024-04-26T04:54:12.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:12 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:12 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:12 smithi005 ceph-mon[91801]: osdmap e499: 8 total, 8 up, 8 in 2024-04-26T04:54:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:12 smithi111 ceph-mon[86360]: osdmap e499: 8 total, 8 up, 8 in 2024-04-26T04:54:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:12 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:12 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:13.047 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-26T04:54:13.408 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:54:13] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2024-04-26T04:54:13.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:13 smithi005 ceph-mon[91801]: osdmap e500: 8 total, 8 up, 8 in 2024-04-26T04:54:13.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:13 smithi005 ceph-mon[91801]: pgmap v379: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s 2024-04-26T04:54:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:13 smithi005 ceph-mon[95809]: osdmap e500: 8 total, 8 up, 8 in 2024-04-26T04:54:13.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:13 smithi005 ceph-mon[95809]: pgmap v379: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s 2024-04-26T04:54:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:13 smithi111 ceph-mon[86360]: osdmap e500: 8 total, 8 up, 8 in 2024-04-26T04:54:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:13 smithi111 ceph-mon[86360]: pgmap v379: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s 2024-04-26T04:54:14.594 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:14.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:14 smithi005 ceph-mon[91801]: osdmap e501: 8 total, 8 up, 8 in 2024-04-26T04:54:14.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:14 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:14.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:14 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:14.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:14 smithi005 ceph-mon[95809]: osdmap e501: 8 total, 8 up, 8 in 2024-04-26T04:54:14.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:14 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:14.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:14 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:14 smithi111 ceph-mon[86360]: osdmap e501: 8 total, 8 up, 8 in 2024-04-26T04:54:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:14 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:14 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: osdmap e502: 8 total, 8 up, 8 in 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: pgmap v382: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:54:15.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: osdmap e502: 8 total, 8 up, 8 in 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: pgmap v382: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:54:15.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:54:15.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:54:15.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:15.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:15.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:54:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: osdmap e502: 8 total, 8 up, 8 in 2024-04-26T04:54:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: pgmap v382: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s 2024-04-26T04:54:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:16.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:54:16.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: pgmap v383: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: pgmap v384: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: Upgrade: Setting container_image for all mon 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: Upgrade: Setting container_image for all crash 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: Upgrade: osd.0 is safe to restart 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: osdmap e503: 8 total, 8 up, 8 in 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4192963506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: pgmap v383: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-04-26T04:54:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: pgmap v384: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: Upgrade: Setting container_image for all mon 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: Upgrade: Setting container_image for all crash 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: Upgrade: osd.0 is safe to restart 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: osdmap e503: 8 total, 8 up, 8 in 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:16.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:16 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4192963506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: pgmap v383: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-04-26T04:54:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: pgmap v384: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: Upgrade: Setting container_image for all mon 2024-04-26T04:54:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: Upgrade: Setting container_image for all crash 2024-04-26T04:54:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:54:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: Upgrade: osd.0 is safe to restart 2024-04-26T04:54:17.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: osdmap e503: 8 total, 8 up, 8 in 2024-04-26T04:54:17.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:17.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:16 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4192963506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:17.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[91801]: osdmap e504: 8 total, 8 up, 8 in 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[91801]: osdmap e505: 8 total, 8 up, 8 in 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[95809]: osdmap e504: 8 total, 8 up, 8 in 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:17.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:17 smithi005 ceph-mon[95809]: osdmap e505: 8 total, 8 up, 8 in 2024-04-26T04:54:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:17 smithi111 ceph-mon[86360]: osdmap e504: 8 total, 8 up, 8 in 2024-04-26T04:54:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:17 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:54:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:17 smithi111 ceph-mon[86360]: osdmap e505: 8 total, 8 up, 8 in 2024-04-26T04:54:18.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[91801]: pgmap v387: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[91801]: Upgrade: Updating osd.0 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[91801]: Deploying daemon osd.0 on smithi005 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[91801]: osdmap e506: 8 total, 8 up, 8 in 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[95809]: pgmap v387: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[95809]: Upgrade: Updating osd.0 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[95809]: Deploying daemon osd.0 on smithi005 2024-04-26T04:54:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:18 smithi005 ceph-mon[95809]: osdmap e506: 8 total, 8 up, 8 in 2024-04-26T04:54:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:18 smithi111 ceph-mon[86360]: pgmap v387: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:18 smithi111 ceph-mon[86360]: Upgrade: Updating osd.0 2024-04-26T04:54:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:18 smithi111 ceph-mon[86360]: Deploying daemon osd.0 on smithi005 2024-04-26T04:54:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:18 smithi111 ceph-mon[86360]: osdmap e506: 8 total, 8 up, 8 in 2024-04-26T04:54:19.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:19 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/232949841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:19.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:19 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/232949841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:20.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:19 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/232949841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:20.942 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:20 smithi005 ceph-mon[91801]: pgmap v390: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-04-26T04:54:20.943 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:20 smithi005 ceph-mon[91801]: osdmap e507: 8 total, 8 up, 8 in 2024-04-26T04:54:20.943 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:20 smithi005 ceph-mon[95809]: pgmap v390: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-04-26T04:54:20.943 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:20 smithi005 ceph-mon[95809]: osdmap e507: 8 total, 8 up, 8 in 2024-04-26T04:54:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:20 smithi111 ceph-mon[86360]: pgmap v390: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-04-26T04:54:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:20 smithi111 ceph-mon[86360]: osdmap e507: 8 total, 8 up, 8 in 2024-04-26T04:54:21.204 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:20 smithi005 systemd[1]: Stopping Ceph osd.0 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:54:21.479 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[40912]: 2024-04-26T04:54:21.202+0000 7efe91c49700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:54:21.479 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[40912]: 2024-04-26T04:54:21.202+0000 7efe91c49700 -1 osd.0 508 *** Got signal Terminated *** 2024-04-26T04:54:21.480 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[40912]: 2024-04-26T04:54:21.202+0000 7efe91c49700 -1 osd.0 508 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:54:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-mon[91801]: osdmap e508: 8 total, 8 up, 8 in 2024-04-26T04:54:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-mon[91801]: osd.0 marked itself down and dead 2024-04-26T04:54:21.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3414773924' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-mon[95809]: osdmap e508: 8 total, 8 up, 8 in 2024-04-26T04:54:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-mon[95809]: osd.0 marked itself down and dead 2024-04-26T04:54:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:21 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3414773924' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:21 smithi111 ceph-mon[86360]: osdmap e508: 8 total, 8 up, 8 in 2024-04-26T04:54:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:21 smithi111 ceph-mon[86360]: osd.0 marked itself down and dead 2024-04-26T04:54:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:21 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3414773924' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:22.556 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:22 smithi005 podman[104276]: 2024-04-26 04:54:22.255511953 +0000 UTC m=+1.165180062 container died b99cc094c6d760874140148ed4d54a7b3e89024cf00ee9640b03b66841dcfc04 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T04:54:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:22 smithi005 ceph-mon[91801]: pgmap v393: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-04-26T04:54:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:22 smithi005 ceph-mon[91801]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:22 smithi005 ceph-mon[91801]: osdmap e509: 8 total, 7 up, 8 in 2024-04-26T04:54:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:22 smithi005 ceph-mon[95809]: pgmap v393: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-04-26T04:54:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:22 smithi005 ceph-mon[95809]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:22.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:22 smithi005 ceph-mon[95809]: osdmap e509: 8 total, 7 up, 8 in 2024-04-26T04:54:22.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:22 smithi005 podman[104276]: 2024-04-26 04:54:22.753567533 +0000 UTC m=+1.663235684 container cleanup b99cc094c6d760874140148ed4d54a7b3e89024cf00ee9640b03b66841dcfc04 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:54:22.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:22 smithi005 bash[104276]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0 2024-04-26T04:54:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:22 smithi111 ceph-mon[86360]: pgmap v393: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-04-26T04:54:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:22 smithi111 ceph-mon[86360]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:22 smithi111 ceph-mon[86360]: osdmap e509: 8 total, 7 up, 8 in 2024-04-26T04:54:23.330 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104314]: 2024-04-26 04:54:23.023150985 +0000 UTC m=+0.758581495 container remove b99cc094c6d760874140148ed4d54a7b3e89024cf00ee9640b03b66841dcfc04 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T04:54:23.330 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:54:23] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2024-04-26T04:54:23.601 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104410]: 2024-04-26 04:54:23.233739075 +0000 UTC m=+0.018725467 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:23.601 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104410]: 2024-04-26 04:54:23.34790497 +0000 UTC m=+0.132891360 container create f553f3d3767376227b49a830879608e96c5bcedc2b2a20e8ffd23b498468270f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:54:23.602 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104410]: 2024-04-26 04:54:23.600616176 +0000 UTC m=+0.385602569 container init f553f3d3767376227b49a830879608e96c5bcedc2b2a20e8ffd23b498468270f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T04:54:23.929 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:23 smithi005 ceph-mon[95809]: osdmap e510: 8 total, 7 up, 8 in 2024-04-26T04:54:23.929 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:23 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3295180856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:23.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:23 smithi005 ceph-mon[91801]: osdmap e510: 8 total, 7 up, 8 in 2024-04-26T04:54:23.929 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:23 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3295180856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:23.929 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104410]: 2024-04-26 04:54:23.604420469 +0000 UTC m=+0.389406868 container start f553f3d3767376227b49a830879608e96c5bcedc2b2a20e8ffd23b498468270f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T04:54:23.930 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104410]: 2024-04-26 04:54:23.66723576 +0000 UTC m=+0.452222160 container attach f553f3d3767376227b49a830879608e96c5bcedc2b2a20e8ffd23b498468270f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:54:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:23 smithi111 ceph-mon[86360]: osdmap e510: 8 total, 7 up, 8 in 2024-04-26T04:54:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:23 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3295180856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:24.229 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:23 smithi005 podman[104410]: 2024-04-26 04:54:23.929205901 +0000 UTC m=+0.714192301 container died f553f3d3767376227b49a830879608e96c5bcedc2b2a20e8ffd23b498468270f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb) 2024-04-26T04:54:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:24 smithi005 ceph-mon[91801]: pgmap v396: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-04-26T04:54:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:24 smithi005 ceph-mon[91801]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-04-26T04:54:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:24 smithi005 ceph-mon[91801]: osdmap e511: 8 total, 7 up, 8 in 2024-04-26T04:54:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:24 smithi005 ceph-mon[95809]: pgmap v396: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-04-26T04:54:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:24 smithi005 ceph-mon[95809]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-04-26T04:54:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:24 smithi005 ceph-mon[95809]: osdmap e511: 8 total, 7 up, 8 in 2024-04-26T04:54:24.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:24 smithi005 podman[104462]: 2024-04-26 04:54:24.679949142 +0000 UTC m=+0.745491341 container remove f553f3d3767376227b49a830879608e96c5bcedc2b2a20e8ffd23b498468270f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb) 2024-04-26T04:54:24.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:24 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T04:54:24.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:24 smithi005 systemd[1]: Stopped Ceph osd.0 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:54:24.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:24 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.0.service: Consumed 26.994s CPU time. 2024-04-26T04:54:24.981 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:24 smithi005 systemd[1]: Starting Ceph osd.0 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:54:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:24 smithi111 ceph-mon[86360]: pgmap v396: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-04-26T04:54:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:24 smithi111 ceph-mon[86360]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-04-26T04:54:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:24 smithi111 ceph-mon[86360]: osdmap e511: 8 total, 7 up, 8 in 2024-04-26T04:54:25.400 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 podman[104565]: 2024-04-26 04:54:25.047551182 +0000 UTC m=+0.018461131 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:25.400 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 podman[104565]: 2024-04-26 04:54:25.147788458 +0000 UTC m=+0.118698392 container create 68e6b1e0a6cbe4d413b8c2cca2785bc1203862a771f872c4aedca3a7e187785b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T04:54:25.729 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 podman[104565]: 2024-04-26 04:54:25.40026319 +0000 UTC m=+0.371173124 container init 68e6b1e0a6cbe4d413b8c2cca2785bc1203862a771f872c4aedca3a7e187785b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:54:25.729 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 podman[104565]: 2024-04-26 04:54:25.403769307 +0000 UTC m=+0.374679250 container start 68e6b1e0a6cbe4d413b8c2cca2785bc1203862a771f872c4aedca3a7e187785b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T04:54:25.730 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 podman[104565]: 2024-04-26 04:54:25.475206949 +0000 UTC m=+0.446116884 container attach 68e6b1e0a6cbe4d413b8c2cca2785bc1203862a771f872c4aedca3a7e187785b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:54:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:25 smithi111 ceph-mon[86360]: osdmap e512: 8 total, 7 up, 8 in 2024-04-26T04:54:26.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-mon[91801]: osdmap e512: 8 total, 7 up, 8 in 2024-04-26T04:54:26.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-mon[95809]: osdmap e512: 8 total, 7 up, 8 in 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 bash[104565]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:54:26.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T04:54:26.231 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:54:26.231 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate[104579]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T04:54:26.231 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:25 smithi005 podman[104565]: 2024-04-26 04:54:25.91652247 +0000 UTC m=+0.887432406 container died 68e6b1e0a6cbe4d413b8c2cca2785bc1203862a771f872c4aedca3a7e187785b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T04:54:26.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:26 smithi005 ceph-mon[91801]: pgmap v399: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-04-26T04:54:26.755 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:26 smithi005 ceph-mon[91801]: osdmap e513: 8 total, 7 up, 8 in 2024-04-26T04:54:26.756 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:26 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1962778567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:26.756 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:26 smithi005 ceph-mon[95809]: pgmap v399: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-04-26T04:54:26.756 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:26 smithi005 ceph-mon[95809]: osdmap e513: 8 total, 7 up, 8 in 2024-04-26T04:54:26.756 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:26 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1962778567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:26.756 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:26 smithi005 podman[104719]: 2024-04-26 04:54:26.495851683 +0000 UTC m=+0.573752176 container remove 68e6b1e0a6cbe4d413b8c2cca2785bc1203862a771f872c4aedca3a7e187785b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T04:54:26.756 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:26 smithi005 podman[104783]: 2024-04-26 04:54:26.626915258 +0000 UTC m=+0.019050386 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:26 smithi111 ceph-mon[86360]: pgmap v399: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-04-26T04:54:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:26 smithi111 ceph-mon[86360]: osdmap e513: 8 total, 7 up, 8 in 2024-04-26T04:54:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:26 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1962778567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:27.024 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:26 smithi005 podman[104783]: 2024-04-26 04:54:26.755093965 +0000 UTC m=+0.147229073 container create d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T04:54:27.479 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:27 smithi005 podman[104783]: 2024-04-26 04:54:27.024411831 +0000 UTC m=+0.416546940 container init d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T04:54:27.480 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:27 smithi005 podman[104783]: 2024-04-26 04:54:27.027669274 +0000 UTC m=+0.419804392 container start d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T04:54:27.480 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-osd[104803]: -- 172.21.15.5:0/2826188998 <== mon.0 v2:172.21.15.5:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5617b2cbd680 con 0x5617b2c87000 2024-04-26T04:54:27.480 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:27 smithi005 bash[104783]: d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e 2024-04-26T04:54:27.480 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:27 smithi005 systemd[1]: Started Ceph osd.0 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:54:27.980 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[104799]: 2024-04-26T04:54:27.572+0000 7ff39cabd7c0 -1 Falling back to public interface 2024-04-26T04:54:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[91801]: osdmap e514: 8 total, 7 up, 8 in 2024-04-26T04:54:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:27.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[91801]: osdmap e515: 8 total, 7 up, 8 in 2024-04-26T04:54:27.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[95809]: osdmap e514: 8 total, 7 up, 8 in 2024-04-26T04:54:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:27.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:27 smithi005 ceph-mon[95809]: osdmap e515: 8 total, 7 up, 8 in 2024-04-26T04:54:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:27 smithi111 ceph-mon[86360]: osdmap e514: 8 total, 7 up, 8 in 2024-04-26T04:54:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:27 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:27 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:27 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:27 smithi111 ceph-mon[86360]: osdmap e515: 8 total, 7 up, 8 in 2024-04-26T04:54:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:28 smithi111 ceph-mon[86360]: pgmap v402: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:28 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1926153821' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:29.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:28 smithi005 ceph-mon[95809]: pgmap v402: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:28 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1926153821' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:28 smithi005 ceph-mon[91801]: pgmap v402: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:28 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1926153821' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:29.815 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestMirroring.AioSnapshotCreate (32108 ms) 2024-04-26T04:54:29.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:29 smithi005 ceph-mon[95809]: osdmap e516: 8 total, 7 up, 8 in 2024-04-26T04:54:29.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:29 smithi005 ceph-mon[91801]: osdmap e516: 8 total, 7 up, 8 in 2024-04-26T04:54:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:29 smithi111 ceph-mon[86360]: osdmap e516: 8 total, 7 up, 8 in 2024-04-26T04:54:30.472 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[104799]: 2024-04-26T04:54:30.385+0000 7ff39cabd7c0 -1 osd.0 508 log_to_monitors true 2024-04-26T04:54:30.815 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 51 tests from TestMirroring (88234 ms total) 2024-04-26T04:54:30.815 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:54:30.815 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 34 tests from TestInternal 2024-04-26T04:54:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: pgmap v405: 105 pgs: 23 active+undersized+degraded, 19 active+undersized, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 6.5 KiB/s wr, 112 op/s; 57/327 objects degraded (17.431%) 2024-04-26T04:54:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: Health check failed: Degraded data redundancy: 57/327 objects degraded (17.431%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-04-26T04:54:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: osdmap e517: 8 total, 7 up, 8 in 2024-04-26T04:54:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]: dispatch 2024-04-26T04:54:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]: dispatch 2024-04-26T04:54:31.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]': finished 2024-04-26T04:54:31.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: from='osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:54:31.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:54:31.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:30 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/779156742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: pgmap v405: 105 pgs: 23 active+undersized+degraded, 19 active+undersized, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 6.5 KiB/s wr, 112 op/s; 57/327 objects degraded (17.431%) 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: Health check failed: Degraded data redundancy: 57/327 objects degraded (17.431%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: osdmap e517: 8 total, 7 up, 8 in 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]: dispatch 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]: dispatch 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]': finished 2024-04-26T04:54:31.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: from='osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/779156742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: pgmap v405: 105 pgs: 23 active+undersized+degraded, 19 active+undersized, 63 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 6.5 KiB/s wr, 112 op/s; 57/327 objects degraded (17.431%) 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: Health check failed: Degraded data redundancy: 57/327 objects degraded (17.431%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: osdmap e517: 8 total, 7 up, 8 in 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3670143040' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]: dispatch 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]: dispatch 2024-04-26T04:54:31.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa36db27-d904-4154-a842-dde168725bae"}]': finished 2024-04-26T04:54:31.137 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: from='osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:54:31.137 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:54:31.137 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:30 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/779156742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:31.479 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[104799]: 2024-04-26T04:54:31.133+0000 7ff389f0d700 -1 osd.0 508 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: osdmap e518: 8 total, 7 up, 8 in 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='osd.0 ' entity='osd.0' 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: osdmap e518: 8 total, 7 up, 8 in 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='osd.0 ' entity='osd.0' 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: osdmap e518: 8 total, 7 up, 8 in 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='osd.0 ' entity='osd.0' 2024-04-26T04:54:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:32.817 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.OpenByID 2024-04-26T04:54:32.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: pgmap v408: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:54:32.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:54:32.919 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951] boot 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: osdmap e519: 8 total, 8 up, 8 in 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3083125336' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: pgmap v408: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951] boot 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: osdmap e519: 8 total, 8 up, 8 in 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:54:32.920 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3083125336' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:32.921 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:32.921 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:32.921 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:32 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: pgmap v408: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: osd.0 [v2:172.21.15.5:6802/752873951,v1:172.21.15.5:6803/752873951] boot 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: osdmap e519: 8 total, 8 up, 8 in 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3083125336' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:32 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:54:33] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.43.0" 2024-04-26T04:54:33.878 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:33.878 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-mon[91801]: osdmap e520: 8 total, 8 up, 8 in 2024-04-26T04:54:33.878 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2240684904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:33.879 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:33.879 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-mon[95809]: osdmap e520: 8 total, 8 up, 8 in 2024-04-26T04:54:33.879 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:33 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2240684904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:34.101 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:33 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:34.101 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:33 smithi111 ceph-mon[86360]: osdmap e520: 8 total, 8 up, 8 in 2024-04-26T04:54:34.101 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:33 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2240684904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:34.859 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.OpenByID (2041 ms) 2024-04-26T04:54:34.859 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-04-26T04:54:34.882 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.OpenSnapDNE (24 ms) 2024-04-26T04:54:34.882 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-04-26T04:54:34.905 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (24 ms) 2024-04-26T04:54:34.906 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-04-26T04:54:34.928 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.ResizeLocksImage (22 ms) 2024-04-26T04:54:34.928 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-04-26T04:54:34.946 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (18 ms) 2024-04-26T04:54:34.946 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-04-26T04:54:34.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:34 smithi005 ceph-mon[91801]: pgmap v411: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:34.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:34 smithi005 ceph-mon[91801]: osdmap e521: 8 total, 8 up, 8 in 2024-04-26T04:54:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:34 smithi005 ceph-mon[95809]: pgmap v411: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:34 smithi005 ceph-mon[95809]: osdmap e521: 8 total, 8 up, 8 in 2024-04-26T04:54:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:34 smithi111 ceph-mon[86360]: pgmap v411: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:34 smithi111 ceph-mon[86360]: osdmap e521: 8 total, 8 up, 8 in 2024-04-26T04:54:35.729 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:54:35.662+0000 7f15dde08700 -1 mgr.server reply reply (11) Resource temporarily unavailable 6 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:54:35.730 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:54:35.662+0000 7f15dde08700 -1 mgr.server reply reply (16) Device or resource busy 6 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: osdmap e522: 8 total, 8 up, 8 in 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1358848082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: osdmap e522: 8 total, 8 up, 8 in 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1358848082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:36.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:35 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: osdmap e522: 8 total, 8 up, 8 in 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1358848082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:36.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:35 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:36.851 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1905 ms) 2024-04-26T04:54:36.851 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-04-26T04:54:36.878 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (27 ms) 2024-04-26T04:54:36.879 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-04-26T04:54:37.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[95809]: pgmap v414: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[95809]: pgmap v415: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[95809]: pgmap v416: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[95809]: Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[95809]: osdmap e523: 8 total, 8 up, 8 in 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[91801]: pgmap v414: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[91801]: pgmap v415: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[91801]: pgmap v416: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:37.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[91801]: Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:54:37.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:36 smithi005 ceph-mon[91801]: osdmap e523: 8 total, 8 up, 8 in 2024-04-26T04:54:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:36 smithi111 ceph-mon[86360]: pgmap v414: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:36 smithi111 ceph-mon[86360]: pgmap v415: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:36 smithi111 ceph-mon[86360]: pgmap v416: 105 pgs: 26 creating+peering, 6 unknown, 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:54:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:36 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:36 smithi111 ceph-mon[86360]: Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:54:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:36 smithi111 ceph-mon[86360]: osdmap e523: 8 total, 8 up, 8 in 2024-04-26T04:54:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:37 smithi005 ceph-mon[95809]: osdmap e524: 8 total, 8 up, 8 in 2024-04-26T04:54:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:37 smithi005 ceph-mon[95809]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:37 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2290903271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:37 smithi005 ceph-mon[91801]: osdmap e524: 8 total, 8 up, 8 in 2024-04-26T04:54:38.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:37 smithi005 ceph-mon[91801]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:38.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:37 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2290903271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:37 smithi111 ceph-mon[86360]: osdmap e524: 8 total, 8 up, 8 in 2024-04-26T04:54:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:37 smithi111 ceph-mon[86360]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:37 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2290903271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:38.865 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1986 ms) 2024-04-26T04:54:38.865 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-04-26T04:54:39.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:38 smithi005 ceph-mon[91801]: pgmap v419: 105 pgs: 26 creating+peering, 15 peering, 64 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 2.2 KiB/s wr, 40 op/s 2024-04-26T04:54:39.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:38 smithi005 ceph-mon[91801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T04:54:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:38 smithi005 ceph-mon[91801]: osdmap e525: 8 total, 8 up, 8 in 2024-04-26T04:54:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:38 smithi005 ceph-mon[95809]: pgmap v419: 105 pgs: 26 creating+peering, 15 peering, 64 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 2.2 KiB/s wr, 40 op/s 2024-04-26T04:54:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:38 smithi005 ceph-mon[95809]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T04:54:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:38 smithi005 ceph-mon[95809]: osdmap e525: 8 total, 8 up, 8 in 2024-04-26T04:54:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:38 smithi111 ceph-mon[86360]: pgmap v419: 105 pgs: 26 creating+peering, 15 peering, 64 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 2.2 KiB/s wr, 40 op/s 2024-04-26T04:54:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:38 smithi111 ceph-mon[86360]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T04:54:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:38 smithi111 ceph-mon[86360]: osdmap e525: 8 total, 8 up, 8 in 2024-04-26T04:54:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:39 smithi005 ceph-mon[91801]: osdmap e526: 8 total, 8 up, 8 in 2024-04-26T04:54:40.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:39 smithi005 ceph-mon[95809]: osdmap e526: 8 total, 8 up, 8 in 2024-04-26T04:54:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:39 smithi111 ceph-mon[86360]: osdmap e526: 8 total, 8 up, 8 in 2024-04-26T04:54:40.873 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2008 ms) 2024-04-26T04:54:40.874 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-04-26T04:54:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:40 smithi005 ceph-mon[91801]: pgmap v422: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 10 KiB/s wr, 119 op/s 2024-04-26T04:54:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:40 smithi005 ceph-mon[91801]: osdmap e527: 8 total, 8 up, 8 in 2024-04-26T04:54:41.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:40 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3063059853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:40 smithi005 ceph-mon[95809]: pgmap v422: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 10 KiB/s wr, 119 op/s 2024-04-26T04:54:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:40 smithi005 ceph-mon[95809]: osdmap e527: 8 total, 8 up, 8 in 2024-04-26T04:54:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:40 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3063059853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:40 smithi111 ceph-mon[86360]: pgmap v422: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 10 KiB/s wr, 119 op/s 2024-04-26T04:54:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:40 smithi111 ceph-mon[86360]: osdmap e527: 8 total, 8 up, 8 in 2024-04-26T04:54:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:40 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3063059853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:42.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:41 smithi005 ceph-mon[91801]: osdmap e528: 8 total, 8 up, 8 in 2024-04-26T04:54:42.130 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:41 smithi005 ceph-mon[95809]: osdmap e528: 8 total, 8 up, 8 in 2024-04-26T04:54:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:41 smithi111 ceph-mon[86360]: osdmap e528: 8 total, 8 up, 8 in 2024-04-26T04:54:42.673 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapSetReleasesLock (1800 ms) 2024-04-26T04:54:42.673 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-04-26T04:54:43.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[91801]: pgmap v425: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 12 KiB/s wr, 137 op/s 2024-04-26T04:54:43.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[91801]: osdmap e529: 8 total, 8 up, 8 in 2024-04-26T04:54:43.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3773957216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:43.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[91801]: osdmap e530: 8 total, 8 up, 8 in 2024-04-26T04:54:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[95809]: pgmap v425: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 12 KiB/s wr, 137 op/s 2024-04-26T04:54:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[95809]: osdmap e529: 8 total, 8 up, 8 in 2024-04-26T04:54:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3773957216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:42 smithi005 ceph-mon[95809]: osdmap e530: 8 total, 8 up, 8 in 2024-04-26T04:54:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:42 smithi111 ceph-mon[86360]: pgmap v425: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 12 KiB/s wr, 137 op/s 2024-04-26T04:54:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:42 smithi111 ceph-mon[86360]: osdmap e529: 8 total, 8 up, 8 in 2024-04-26T04:54:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:42 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3773957216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:42 smithi111 ceph-mon[86360]: osdmap e530: 8 total, 8 up, 8 in 2024-04-26T04:54:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:54:43] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.43.0" 2024-04-26T04:54:44.685 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.FlattenLocksImage (2011 ms) 2024-04-26T04:54:44.685 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-04-26T04:54:44.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:44 smithi005 ceph-mon[91801]: pgmap v428: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 7.5 KiB/s wr, 121 op/s 2024-04-26T04:54:44.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:44 smithi005 ceph-mon[91801]: osdmap e531: 8 total, 8 up, 8 in 2024-04-26T04:54:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:44 smithi005 ceph-mon[95809]: pgmap v428: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 7.5 KiB/s wr, 121 op/s 2024-04-26T04:54:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:44 smithi005 ceph-mon[95809]: osdmap e531: 8 total, 8 up, 8 in 2024-04-26T04:54:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:44 smithi111 ceph-mon[86360]: pgmap v428: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 7.5 KiB/s wr, 121 op/s 2024-04-26T04:54:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:44 smithi111 ceph-mon[86360]: osdmap e531: 8 total, 8 up, 8 in 2024-04-26T04:54:45.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:45 smithi005 ceph-mon[91801]: osdmap e532: 8 total, 8 up, 8 in 2024-04-26T04:54:45.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:45 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2780895595' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:45.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:45 smithi005 ceph-mon[95809]: osdmap e532: 8 total, 8 up, 8 in 2024-04-26T04:54:45.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:45 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2780895595' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:45 smithi111 ceph-mon[86360]: osdmap e532: 8 total, 8 up, 8 in 2024-04-26T04:54:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:45 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2780895595' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:46.681 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (1997 ms) 2024-04-26T04:54:46.681 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-04-26T04:54:46.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[91801]: pgmap v431: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 4.7 KiB/s wr, 76 op/s 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[91801]: osdmap e533: 8 total, 8 up, 8 in 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[95809]: pgmap v431: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 4.7 KiB/s wr, 76 op/s 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[95809]: osdmap e533: 8 total, 8 up, 8 in 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:46 smithi111 ceph-mon[86360]: pgmap v431: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 4.7 KiB/s wr, 76 op/s 2024-04-26T04:54:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:46 smithi111 ceph-mon[86360]: osdmap e533: 8 total, 8 up, 8 in 2024-04-26T04:54:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:47.687 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (1006 ms) 2024-04-26T04:54:47.688 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-04-26T04:54:47.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[91801]: osdmap e534: 8 total, 8 up, 8 in 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2708411967' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]: dispatch 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]: dispatch 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1933838119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[95809]: osdmap e534: 8 total, 8 up, 8 in 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2708411967' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]: dispatch 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]: dispatch 2024-04-26T04:54:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:47 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1933838119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:47 smithi111 ceph-mon[86360]: osdmap e534: 8 total, 8 up, 8 in 2024-04-26T04:54:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:47 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2708411967' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]: dispatch 2024-04-26T04:54:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:47 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]: dispatch 2024-04-26T04:54:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:47 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1933838119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:48.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[91801]: pgmap v434: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]': finished 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[91801]: osdmap e535: 8 total, 8 up, 8 in 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/209732525' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]: dispatch 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]: dispatch 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[95809]: pgmap v434: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]': finished 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[95809]: osdmap e535: 8 total, 8 up, 8 in 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/209732525' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]: dispatch 2024-04-26T04:54:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:48 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]: dispatch 2024-04-26T04:54:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:48 smithi111 ceph-mon[86360]: pgmap v434: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:54:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:48 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/2708411967"}]': finished 2024-04-26T04:54:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:48 smithi111 ceph-mon[86360]: osdmap e535: 8 total, 8 up, 8 in 2024-04-26T04:54:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:48 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/209732525' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]: dispatch 2024-04-26T04:54:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:48 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]: dispatch 2024-04-26T04:54:49.686 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (1999 ms) 2024-04-26T04:54:49.687 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-04-26T04:54:49.715 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.AioWriteRequestsLock (29 ms) 2024-04-26T04:54:49.716 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-04-26T04:54:49.742 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (27 ms) 2024-04-26T04:54:49.742 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-04-26T04:54:49.782 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.CancelAsyncResize (40 ms) 2024-04-26T04:54:49.783 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.MultipleResize 2024-04-26T04:54:49.835 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.MultipleResize (53 ms) 2024-04-26T04:54:49.835 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.Metadata 2024-04-26T04:54:49.879 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.Metadata (44 ms) 2024-04-26T04:54:49.879 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-04-26T04:54:49.907 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.MetadataConfApply (28 ms) 2024-04-26T04:54:49.907 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-04-26T04:54:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:49 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]': finished 2024-04-26T04:54:49.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:49 smithi005 ceph-mon[91801]: osdmap e536: 8 total, 8 up, 8 in 2024-04-26T04:54:49.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:49 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2298914066' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:49 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]': finished 2024-04-26T04:54:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:49 smithi005 ceph-mon[95809]: osdmap e536: 8 total, 8 up, 8 in 2024-04-26T04:54:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:49 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2298914066' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:49 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/209732525"}]': finished 2024-04-26T04:54:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:49 smithi111 ceph-mon[86360]: osdmap e536: 8 total, 8 up, 8 in 2024-04-26T04:54:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:49 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2298914066' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:50.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:50 smithi005 ceph-mon[95809]: pgmap v437: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 12 KiB/s wr, 245 op/s 2024-04-26T04:54:50.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:50 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:50.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:50 smithi005 ceph-mon[91801]: pgmap v437: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 12 KiB/s wr, 245 op/s 2024-04-26T04:54:50.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:50 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:50 smithi111 ceph-mon[86360]: pgmap v437: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 12 KiB/s wr, 245 op/s 2024-04-26T04:54:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:50 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:51 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:51 smithi005 ceph-mon[95809]: Upgrade: osd.1 is safe to restart 2024-04-26T04:54:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:51 smithi005 ceph-mon[95809]: osdmap e537: 8 total, 8 up, 8 in 2024-04-26T04:54:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:51 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:51 smithi005 ceph-mon[91801]: Upgrade: osd.1 is safe to restart 2024-04-26T04:54:51.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:51 smithi005 ceph-mon[91801]: osdmap e537: 8 total, 8 up, 8 in 2024-04-26T04:54:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:51 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:51 smithi111 ceph-mon[86360]: Upgrade: osd.1 is safe to restart 2024-04-26T04:54:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:51 smithi111 ceph-mon[86360]: osdmap e537: 8 total, 8 up, 8 in 2024-04-26T04:54:52.735 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapshotCopyup (2828 ms) 2024-04-26T04:54:52.735 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-04-26T04:54:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: pgmap v439: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 12 KiB/s wr, 224 op/s 2024-04-26T04:54:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: osdmap e538: 8 total, 8 up, 8 in 2024-04-26T04:54:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4053972722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[91801]: osdmap e539: 8 total, 8 up, 8 in 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: pgmap v439: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 12 KiB/s wr, 224 op/s 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: osdmap e538: 8 total, 8 up, 8 in 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4053972722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:52 smithi005 ceph-mon[95809]: osdmap e539: 8 total, 8 up, 8 in 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: pgmap v439: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 12 KiB/s wr, 224 op/s 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: osdmap e538: 8 total, 8 up, 8 in 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4053972722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:52 smithi111 ceph-mon[86360]: osdmap e539: 8 total, 8 up, 8 in 2024-04-26T04:54:53.463 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:54:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:54:53] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2024-04-26T04:54:53.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:53 smithi005 ceph-mon[91801]: Upgrade: Updating osd.1 2024-04-26T04:54:53.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:53 smithi005 ceph-mon[91801]: Deploying daemon osd.1 on smithi005 2024-04-26T04:54:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:53 smithi005 ceph-mon[95809]: Upgrade: Updating osd.1 2024-04-26T04:54:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:53 smithi005 ceph-mon[95809]: Deploying daemon osd.1 on smithi005 2024-04-26T04:54:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:53 smithi111 ceph-mon[86360]: Upgrade: Updating osd.1 2024-04-26T04:54:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:53 smithi111 ceph-mon[86360]: Deploying daemon osd.1 on smithi005 2024-04-26T04:54:54.737 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (2002 ms) 2024-04-26T04:54:54.738 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-04-26T04:54:54.870 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:54 smithi005 ceph-mon[95809]: pgmap v442: 105 pgs: 105 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 22 KiB/s wr, 222 op/s 2024-04-26T04:54:54.870 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:54 smithi005 ceph-mon[95809]: osdmap e540: 8 total, 8 up, 8 in 2024-04-26T04:54:54.870 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3395284409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:54.870 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:54 smithi005 ceph-mon[91801]: pgmap v442: 105 pgs: 105 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 22 KiB/s wr, 222 op/s 2024-04-26T04:54:54.870 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:54 smithi005 ceph-mon[91801]: osdmap e540: 8 total, 8 up, 8 in 2024-04-26T04:54:54.871 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3395284409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:54 smithi111 ceph-mon[86360]: pgmap v442: 105 pgs: 105 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 22 KiB/s wr, 222 op/s 2024-04-26T04:54:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:54 smithi111 ceph-mon[86360]: osdmap e540: 8 total, 8 up, 8 in 2024-04-26T04:54:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3395284409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:55.745 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (1008 ms) 2024-04-26T04:54:55.745 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-04-26T04:54:55.835 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:55 smithi005 ceph-mon[91801]: osdmap e541: 8 total, 8 up, 8 in 2024-04-26T04:54:55.835 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:55 smithi005 ceph-mon[95809]: osdmap e541: 8 total, 8 up, 8 in 2024-04-26T04:54:55.835 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:55 smithi005 systemd[1]: Stopping Ceph osd.1 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:54:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:55 smithi111 ceph-mon[86360]: osdmap e541: 8 total, 8 up, 8 in 2024-04-26T04:54:56.229 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[46613]: 2024-04-26T04:54:55.833+0000 7face3b28700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:54:56.230 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[46613]: 2024-04-26T04:54:55.833+0000 7face3b28700 -1 osd.1 542 *** Got signal Terminated *** 2024-04-26T04:54:56.230 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:55 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[46613]: 2024-04-26T04:54:55.833+0000 7face3b28700 -1 osd.1 542 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[91801]: pgmap v445: 105 pgs: 105 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 158 KiB/s rd, 24 KiB/s wr, 233 op/s 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[91801]: osdmap e542: 8 total, 8 up, 8 in 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[91801]: osd.1 marked itself down and dead 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1328283822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[95809]: pgmap v445: 105 pgs: 105 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 158 KiB/s rd, 24 KiB/s wr, 233 op/s 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[95809]: osdmap e542: 8 total, 8 up, 8 in 2024-04-26T04:54:56.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[95809]: osd.1 marked itself down and dead 2024-04-26T04:54:56.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:56 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1328283822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:56 smithi111 ceph-mon[86360]: pgmap v445: 105 pgs: 105 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 158 KiB/s rd, 24 KiB/s wr, 233 op/s 2024-04-26T04:54:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:56 smithi111 ceph-mon[86360]: osdmap e542: 8 total, 8 up, 8 in 2024-04-26T04:54:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:56 smithi111 ceph-mon[86360]: osd.1 marked itself down and dead 2024-04-26T04:54:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:56 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1328283822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:57.534 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:57 smithi005 podman[109233]: 2024-04-26 04:54:57.268476672 +0000 UTC m=+1.544398886 container died 9e578b59814273b5de010db0d66c814be29134846c5bcb9d4c0e727c6c78e1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T04:54:57.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:57 smithi005 ceph-mon[91801]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:57 smithi005 ceph-mon[91801]: osdmap e543: 8 total, 7 up, 8 in 2024-04-26T04:54:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:57 smithi005 ceph-mon[91801]: osdmap e544: 8 total, 7 up, 8 in 2024-04-26T04:54:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:57 smithi005 ceph-mon[95809]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:57 smithi005 ceph-mon[95809]: osdmap e543: 8 total, 7 up, 8 in 2024-04-26T04:54:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:57 smithi005 ceph-mon[95809]: osdmap e544: 8 total, 7 up, 8 in 2024-04-26T04:54:57.980 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:57 smithi005 podman[109233]: 2024-04-26 04:54:57.726206118 +0000 UTC m=+2.002128320 container cleanup 9e578b59814273b5de010db0d66c814be29134846c5bcb9d4c0e727c6c78e1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T04:54:57.980 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:57 smithi005 bash[109233]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1 2024-04-26T04:54:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:57 smithi111 ceph-mon[86360]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:57 smithi111 ceph-mon[86360]: osdmap e543: 8 total, 7 up, 8 in 2024-04-26T04:54:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:57 smithi111 ceph-mon[86360]: osdmap e544: 8 total, 7 up, 8 in 2024-04-26T04:54:58.360 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109286]: 2024-04-26 04:54:58.02772167 +0000 UTC m=+0.756117129 container remove 9e578b59814273b5de010db0d66c814be29134846c5bcb9d4c0e727c6c78e1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:54:58.615 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109376]: 2024-04-26 04:54:58.263643544 +0000 UTC m=+0.018471348 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:58.615 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109376]: 2024-04-26 04:54:58.378240551 +0000 UTC m=+0.133068336 container create 55dd9232ad238fe7e8d396fe2471f8dfeb8669e6677e1c2dffae31ff3abf971f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T04:54:58.615 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109376]: 2024-04-26 04:54:58.614256918 +0000 UTC m=+0.369084704 container init 55dd9232ad238fe7e8d396fe2471f8dfeb8669e6677e1c2dffae31ff3abf971f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-26T04:54:58.797 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.ResizeCopyup (3052 ms) 2024-04-26T04:54:58.797 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-04-26T04:54:58.807 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:54:58.807 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.DiscardCopyup (10 ms) 2024-04-26T04:54:58.807 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.ImageOptions 2024-04-26T04:54:58.816 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.ImageOptions (9 ms) 2024-04-26T04:54:58.816 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-04-26T04:54:58.937 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:58 smithi005 ceph-mon[91801]: pgmap v448: 105 pgs: 13 stale+active+clean, 92 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 1.5 KiB/s wr, 45 op/s 2024-04-26T04:54:58.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:58 smithi005 ceph-mon[95809]: pgmap v448: 105 pgs: 13 stale+active+clean, 92 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 1.5 KiB/s wr, 45 op/s 2024-04-26T04:54:58.937 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109376]: 2024-04-26 04:54:58.617938956 +0000 UTC m=+0.372766750 container start 55dd9232ad238fe7e8d396fe2471f8dfeb8669e6677e1c2dffae31ff3abf971f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T04:54:58.937 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109376]: 2024-04-26 04:54:58.697609574 +0000 UTC m=+0.452437371 container attach 55dd9232ad238fe7e8d396fe2471f8dfeb8669e6677e1c2dffae31ff3abf971f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb) 2024-04-26T04:54:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:58 smithi111 ceph-mon[86360]: pgmap v448: 105 pgs: 13 stale+active+clean, 92 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 1.5 KiB/s wr, 45 op/s 2024-04-26T04:54:59.229 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:58 smithi005 podman[109376]: 2024-04-26 04:54:58.936815673 +0000 UTC m=+0.691643452 container died 55dd9232ad238fe7e8d396fe2471f8dfeb8669e6677e1c2dffae31ff3abf971f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T04:54:59.748 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 podman[109428]: 2024-04-26 04:54:59.484341227 +0000 UTC m=+0.541839604 container remove 55dd9232ad238fe7e8d396fe2471f8dfeb8669e6677e1c2dffae31ff3abf971f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T04:54:59.749 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T04:54:59.749 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 systemd[1]: Stopped Ceph osd.1 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:54:59.749 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.1.service: Consumed 25.817s CPU time. 2024-04-26T04:54:59.749 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 systemd[1]: Starting Ceph osd.1 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:55:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:59 smithi111 ceph-mon[86360]: osdmap e545: 8 total, 7 up, 8 in 2024-04-26T04:55:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:54:59 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4131215564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:00.068 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:59 smithi005 ceph-mon[91801]: osdmap e545: 8 total, 7 up, 8 in 2024-04-26T04:55:00.068 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:54:59 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4131215564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:00.068 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:59 smithi005 ceph-mon[95809]: osdmap e545: 8 total, 7 up, 8 in 2024-04-26T04:55:00.069 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:54:59 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4131215564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:00.069 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 podman[109531]: 2024-04-26 04:54:59.843371964 +0000 UTC m=+0.018513576 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:55:00.069 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:54:59 smithi005 podman[109531]: 2024-04-26 04:54:59.951948822 +0000 UTC m=+0.127090434 container create 0c74666d2e39dbeb9e042a325e8e5468ce06883870db16c747093be980dd7b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T04:55:00.480 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 podman[109531]: 2024-04-26 04:55:00.213210544 +0000 UTC m=+0.388352153 container init 0c74666d2e39dbeb9e042a325e8e5468ce06883870db16c747093be980dd7b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:55:00.480 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 podman[109531]: 2024-04-26 04:55:00.216494693 +0000 UTC m=+0.391636306 container start 0c74666d2e39dbeb9e042a325e8e5468ce06883870db16c747093be980dd7b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T04:55:00.480 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 podman[109531]: 2024-04-26 04:55:00.288179632 +0000 UTC m=+0.463321242 container attach 0c74666d2e39dbeb9e042a325e8e5468ce06883870db16c747093be980dd7b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:55:00.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-mon[95809]: pgmap v451: 105 pgs: 35 peering, 2 stale+active+clean, 68 active+clean; 600 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 221 KiB/s rd, 21 KiB/s wr, 301 op/s 2024-04-26T04:55:00.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-mon[95809]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-26T04:55:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-mon[95809]: osdmap e546: 8 total, 7 up, 8 in 2024-04-26T04:55:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-mon[91801]: pgmap v451: 105 pgs: 35 peering, 2 stale+active+clean, 68 active+clean; 600 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 221 KiB/s rd, 21 KiB/s wr, 301 op/s 2024-04-26T04:55:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-mon[91801]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-26T04:55:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-mon[91801]: osdmap e546: 8 total, 7 up, 8 in 2024-04-26T04:55:00.980 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:55:00.980 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 bash[109531]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate[109546]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T04:55:00.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:00 smithi005 podman[109531]: 2024-04-26 04:55:00.695763782 +0000 UTC m=+0.870905392 container died 0c74666d2e39dbeb9e042a325e8e5468ce06883870db16c747093be980dd7b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb) 2024-04-26T04:55:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:00 smithi111 ceph-mon[86360]: pgmap v451: 105 pgs: 35 peering, 2 stale+active+clean, 68 active+clean; 600 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 221 KiB/s rd, 21 KiB/s wr, 301 op/s 2024-04-26T04:55:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:00 smithi111 ceph-mon[86360]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-26T04:55:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:00 smithi111 ceph-mon[86360]: osdmap e546: 8 total, 7 up, 8 in 2024-04-26T04:55:01.635 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 podman[109664]: 2024-04-26 04:55:01.292208379 +0000 UTC m=+0.590685737 container remove 0c74666d2e39dbeb9e042a325e8e5468ce06883870db16c747093be980dd7b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T04:55:01.635 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 podman[109753]: 2024-04-26 04:55:01.411439425 +0000 UTC m=+0.018333138 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:55:01.635 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 podman[109753]: 2024-04-26 04:55:01.509828702 +0000 UTC m=+0.116722402 container create d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T04:55:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[91801]: osdmap e547: 8 total, 7 up, 8 in 2024-04-26T04:55:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2213331312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[95809]: osdmap e547: 8 total, 7 up, 8 in 2024-04-26T04:55:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2213331312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:01.980 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 podman[109753]: 2024-04-26 04:55:01.754900128 +0000 UTC m=+0.361793838 container init d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T04:55:01.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 podman[109753]: 2024-04-26 04:55:01.760442123 +0000 UTC m=+0.367335831 container start d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T04:55:01.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 ceph-osd[109772]: -- 172.21.15.5:0/33754641 <== mon.2 v2:172.21.15.5:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d2785bd680 con 0x55d278586c00 2024-04-26T04:55:01.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 bash[109753]: d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd 2024-04-26T04:55:01.981 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:01 smithi005 systemd[1]: Started Ceph osd.1 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:55:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:01 smithi111 ceph-mon[86360]: osdmap e547: 8 total, 7 up, 8 in 2024-04-26T04:55:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:01 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2213331312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:02.729 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[109768]: 2024-04-26T04:55:02.303+0000 7fac7930f7c0 -1 Falling back to public interface 2024-04-26T04:55:02.779 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.WriteFullCopyup (3963 ms) 2024-04-26T04:55:02.779 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-04-26T04:55:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:02 smithi111 ceph-mon[86360]: pgmap v454: 105 pgs: 5 active+undersized+degraded, 35 peering, 3 active+undersized, 62 active+clean; 608 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 26 KiB/s wr, 284 op/s; 13/405 objects degraded (3.210%) 2024-04-26T04:55:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:02 smithi111 ceph-mon[86360]: Health check failed: Degraded data redundancy: 13/405 objects degraded (3.210%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:02 smithi111 ceph-mon[86360]: osdmap e548: 8 total, 7 up, 8 in 2024-04-26T04:55:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:03.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[95809]: pgmap v454: 105 pgs: 5 active+undersized+degraded, 35 peering, 3 active+undersized, 62 active+clean; 608 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 26 KiB/s wr, 284 op/s; 13/405 objects degraded (3.210%) 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[95809]: Health check failed: Degraded data redundancy: 13/405 objects degraded (3.210%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[95809]: osdmap e548: 8 total, 7 up, 8 in 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[91801]: pgmap v454: 105 pgs: 5 active+undersized+degraded, 35 peering, 3 active+undersized, 62 active+clean; 608 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 26 KiB/s wr, 284 op/s; 13/405 objects degraded (3.210%) 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[91801]: Health check failed: Degraded data redundancy: 13/405 objects degraded (3.210%), 5 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[91801]: osdmap e548: 8 total, 7 up, 8 in 2024-04-26T04:55:03.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:03.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:03.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:55:03] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2024-04-26T04:55:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:03 smithi005 ceph-mon[91801]: osdmap e549: 8 total, 7 up, 8 in 2024-04-26T04:55:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:03 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1462957954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:03 smithi005 ceph-mon[95809]: osdmap e549: 8 total, 7 up, 8 in 2024-04-26T04:55:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:03 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1462957954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:03 smithi111 ceph-mon[86360]: osdmap e549: 8 total, 7 up, 8 in 2024-04-26T04:55:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:03 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1462957954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:04.804 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2024 ms) 2024-04-26T04:55:04.804 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.TestCoR 2024-04-26T04:55:04.814 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-04-26T04:55:04.814 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.TestCoR (11 ms) 2024-04-26T04:55:04.814 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout:made parent image "image401": 4194304K (1024 * 4K) 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout:generated random write map: 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout: [7 , 31627 ] 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout: [16 , 68963 ] 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout: [36 , 149980 ] 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout: [59 , 242864 ] 2024-04-26T04:55:05.028 INFO:tasks.workunit.client.0.smithi005.stdout: [81 , 332489 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [83 , 340570 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [110 , 453391 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [115 , 471505 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [119 , 489139 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [121 , 498974 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [126 , 517294 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [132 , 540876 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [149 , 612208 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [152 , 624037 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [153 , 629080 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [154 , 632801 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [161 , 660130 ] 2024-04-26T04:55:05.029 INFO:tasks.workunit.client.0.smithi005.stdout: [162 , 663895 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [174 , 716085 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [190 , 779120 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [204 , 839084 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [218 , 895961 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [222 , 909892 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [225 , 922269 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [231 , 948632 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [241 , 990706 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [253 , 1038992 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [263 , 1077509 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [266 , 1092345 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [281 , 1152837 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [292 , 1197598 ] 2024-04-26T04:55:05.030 INFO:tasks.workunit.client.0.smithi005.stdout: [299 , 1224877 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [319 , 1308109 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [328 , 1344111 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [331 , 1358638 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [332 , 1363357 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [354 , 1452646 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [374 , 1534334 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [377 , 1545598 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [385 , 1579733 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [402 , 1648813 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [409 , 1677511 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [417 , 1710053 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [421 , 1727511 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [425 , 1744044 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [427 , 1749937 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [453 , 1855560 ] 2024-04-26T04:55:05.031 INFO:tasks.workunit.client.0.smithi005.stdout: [463 , 1898150 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [465 , 1906465 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [504 , 2066287 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [509 , 2087576 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [526 , 2157662 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [534 , 2190784 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [539 , 2209339 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [540 , 2213535 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [542 , 2223022 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [545 , 2233800 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [555 , 2273845 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [558 , 2287693 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [561 , 2299494 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [571 , 2342056 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [580 , 2377674 ] 2024-04-26T04:55:05.032 INFO:tasks.workunit.client.0.smithi005.stdout: [583 , 2390450 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [602 , 2467742 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [607 , 2489520 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [609 , 2495160 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [637 , 2609961 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [641 , 2627114 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [642 , 2632217 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [683 , 2800426 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [689 , 2825632 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [709 , 2906458 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [725 , 2970084 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [734 , 3007607 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [742 , 3039544 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [750 , 3075030 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [777 , 3185159 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [779 , 3191547 ] 2024-04-26T04:55:05.033 INFO:tasks.workunit.client.0.smithi005.stdout: [805 , 3298716 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [810 , 3320966 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [826 , 3386137 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [828 , 3393294 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [877 , 3592721 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [883 , 3616851 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [895 , 3668272 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [904 , 3705689 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [916 , 3754275 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [927 , 3798406 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [929 , 3806101 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [945 , 3873231 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [954 , 3908930 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [956 , 3916022 ] 2024-04-26T04:55:05.034 INFO:tasks.workunit.client.0.smithi005.stdout: [971 , 3977546 ] 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: [976 , 4001165 ] 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: [987 , 4043115 ] 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: [997 , 4085449 ] 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: [1014 , 4153889 ] 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: [1020 , 4179578 ] 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout:write data based on random map 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-7 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-16 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-36 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-59 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-81 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-83 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-110 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-115 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-119 2024-04-26T04:55:05.035 INFO:tasks.workunit.client.0.smithi005.stdout: write object-121 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-126 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-132 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-149 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-152 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-153 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-154 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-161 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-162 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-174 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-190 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-204 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-218 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-222 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-225 2024-04-26T04:55:05.036 INFO:tasks.workunit.client.0.smithi005.stdout: write object-231 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-241 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-253 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-263 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-266 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-281 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-292 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-299 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-319 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-328 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-331 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-332 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-354 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-374 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-377 2024-04-26T04:55:05.037 INFO:tasks.workunit.client.0.smithi005.stdout: write object-385 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-402 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-409 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-417 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-421 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-425 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-427 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-453 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-463 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-465 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-504 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-509 2024-04-26T04:55:05.038 INFO:tasks.workunit.client.0.smithi005.stdout: write object-526 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-534 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-539 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-540 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-542 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-545 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-555 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-558 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-561 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-571 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-580 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-583 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-602 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-607 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-609 2024-04-26T04:55:05.039 INFO:tasks.workunit.client.0.smithi005.stdout: write object-637 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-641 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-642 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-683 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-689 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-709 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-725 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-734 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-742 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-750 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-777 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-779 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-805 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-810 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-826 2024-04-26T04:55:05.040 INFO:tasks.workunit.client.0.smithi005.stdout: write object-828 2024-04-26T04:55:05.041 INFO:tasks.workunit.client.0.smithi005.stdout: write object-877 2024-04-26T04:55:05.041 INFO:tasks.workunit.client.0.smithi005.stdout: write object-883 2024-04-26T04:55:05.041 INFO:tasks.workunit.client.0.smithi005.stdout: write object-895 2024-04-26T04:55:05.041 INFO:tasks.workunit.client.0.smithi005.stdout: write object-904 2024-04-26T04:55:05.062 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:04 smithi005 ceph-mon[91801]: pgmap v457: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 269 op/s; 41/453 objects degraded (9.051%) 2024-04-26T04:55:05.062 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:04 smithi005 ceph-mon[91801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-26T04:55:05.062 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:04 smithi005 ceph-mon[91801]: osdmap e550: 8 total, 7 up, 8 in 2024-04-26T04:55:05.063 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:04 smithi005 ceph-mon[95809]: pgmap v457: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 269 op/s; 41/453 objects degraded (9.051%) 2024-04-26T04:55:05.063 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:04 smithi005 ceph-mon[95809]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-26T04:55:05.063 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:04 smithi005 ceph-mon[95809]: osdmap e550: 8 total, 7 up, 8 in 2024-04-26T04:55:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:04 smithi111 ceph-mon[86360]: pgmap v457: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 269 op/s; 41/453 objects degraded (9.051%) 2024-04-26T04:55:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:04 smithi111 ceph-mon[86360]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-26T04:55:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:04 smithi111 ceph-mon[86360]: osdmap e550: 8 total, 7 up, 8 in 2024-04-26T04:55:05.479 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[109768]: 2024-04-26T04:55:05.148+0000 7fac7930f7c0 -1 osd.1 542 log_to_monitors true 2024-04-26T04:55:06.087 INFO:tasks.workunit.client.0.smithi005.stdout: write object-916 2024-04-26T04:55:06.087 INFO:tasks.workunit.client.0.smithi005.stdout: write object-927 2024-04-26T04:55:06.087 INFO:tasks.workunit.client.0.smithi005.stdout: write object-929 2024-04-26T04:55:06.087 INFO:tasks.workunit.client.0.smithi005.stdout: write object-945 2024-04-26T04:55:06.087 INFO:tasks.workunit.client.0.smithi005.stdout: write object-954 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-956 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-971 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-976 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-987 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-997 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-1014 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: write object-1020 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout:verify written data by reading 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout: read object-7 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout:made snapshot "image401@parent_snap" and protect it 2024-04-26T04:55:06.088 INFO:tasks.workunit.client.0.smithi005.stdout:made and opened clone "image402" 2024-04-26T04:55:06.089 INFO:tasks.workunit.client.0.smithi005.stdout:flattening clone: "image402" 2024-04-26T04:55:06.089 INFO:tasks.workunit.client.0.smithi005.stdout:check whether child image has the same set of objects as parent 2024-04-26T04:55:06.089 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1273 ms) 2024-04-26T04:55:06.089 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-04-26T04:55:06.120 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[95809]: osdmap e551: 8 total, 7 up, 8 in 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[95809]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[91801]: osdmap e551: 8 total, 7 up, 8 in 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[91801]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:06.121 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:06.179 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.PoolMetadataConfApply (92 ms) 2024-04-26T04:55:06.179 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.Sparsify 2024-04-26T04:55:06.202 INFO:tasks.workunit.client.0.smithi005.stdout:sparsify_supported=1 2024-04-26T04:55:06.202 INFO:tasks.workunit.client.0.smithi005.stdout:sparse_read_supported=1 2024-04-26T04:55:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.Sparsify (53 ms) 2024-04-26T04:55:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.SparsifyClone 2024-04-26T04:55:06.250 INFO:tasks.workunit.client.0.smithi005.stdout:sparsify_supported=1 2024-04-26T04:55:06.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:05 smithi111 ceph-mon[86360]: osdmap e551: 8 total, 7 up, 8 in 2024-04-26T04:55:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:05 smithi111 ceph-mon[86360]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:55:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:06.482 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[109768]: 2024-04-26T04:55:06.126+0000 7fac6675f700 -1 osd.1 542 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:55:06.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:55:06] ENGINE Bus STOPPING 2024-04-26T04:55:06.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:55:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:55:06.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:55:06] ENGINE Bus STOPPED 2024-04-26T04:55:06.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:55:06] ENGINE Bus STARTING 2024-04-26T04:55:06.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:55:06] ENGINE Serving on http://:::9283 2024-04-26T04:55:06.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:55:06] ENGINE Bus STARTED 2024-04-26T04:55:07.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: pgmap v460: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 240 op/s; 41/453 objects degraded (9.051%) 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: osdmap e552: 8 total, 7 up, 8 in 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1835818858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:07.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: pgmap v460: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 240 op/s; 41/453 objects degraded (9.051%) 2024-04-26T04:55:07.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:55:07.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: osdmap e552: 8 total, 7 up, 8 in 2024-04-26T04:55:07.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:55:07.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1835818858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:07.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' 2024-04-26T04:55:07.232 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' 2024-04-26T04:55:07.232 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:07.232 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: pgmap v460: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 240 op/s; 41/453 objects degraded (9.051%) 2024-04-26T04:55:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:55:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: osdmap e552: 8 total, 7 up, 8 in 2024-04-26T04:55:07.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:55:07.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1835818858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:07.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' 2024-04-26T04:55:07.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551]' entity='osd.1' 2024-04-26T04:55:07.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:07.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:07.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[91801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:55:07.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[91801]: osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551] boot 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[91801]: osdmap e553: 8 total, 8 up, 8 in 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[91801]: Health check update: Degraded data redundancy: 41/453 objects degraded (9.051%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[95809]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[95809]: osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551] boot 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[95809]: osdmap e553: 8 total, 8 up, 8 in 2024-04-26T04:55:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:07 smithi005 ceph-mon[95809]: Health check update: Degraded data redundancy: 41/453 objects degraded (9.051%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:07 smithi111 ceph-mon[86360]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:55:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:55:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:07 smithi111 ceph-mon[86360]: osd.1 [v2:172.21.15.5:6810/3934897551,v1:172.21.15.5:6811/3934897551] boot 2024-04-26T04:55:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:07 smithi111 ceph-mon[86360]: osdmap e553: 8 total, 8 up, 8 in 2024-04-26T04:55:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:07 smithi111 ceph-mon[86360]: Health check update: Degraded data redundancy: 41/453 objects degraded (9.051%), 14 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:09.148 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.SparsifyClone (2916 ms) 2024-04-26T04:55:09.148 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestInternal.MissingDataPool 2024-04-26T04:55:09.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:09 smithi005 ceph-mon[95809]: pgmap v463: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.7 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 165 KiB/s rd, 57 KiB/s wr, 258 op/s; 55/540 objects degraded (10.185%) 2024-04-26T04:55:09.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:09 smithi005 ceph-mon[95809]: osdmap e554: 8 total, 8 up, 8 in 2024-04-26T04:55:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:09 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/506017403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:09 smithi005 ceph-mon[91801]: pgmap v463: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.7 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 165 KiB/s rd, 57 KiB/s wr, 258 op/s; 55/540 objects degraded (10.185%) 2024-04-26T04:55:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:09 smithi005 ceph-mon[91801]: osdmap e554: 8 total, 8 up, 8 in 2024-04-26T04:55:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:09 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/506017403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:09 smithi111 ceph-mon[86360]: pgmap v463: 105 pgs: 14 active+undersized+degraded, 29 active+undersized, 62 active+clean; 4.7 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 165 KiB/s rd, 57 KiB/s wr, 258 op/s; 55/540 objects degraded (10.185%) 2024-04-26T04:55:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:09 smithi111 ceph-mon[86360]: osdmap e554: 8 total, 8 up, 8 in 2024-04-26T04:55:09.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:09 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/506017403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:10.266 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestInternal.MissingDataPool (1118 ms) 2024-04-26T04:55:10.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:10 smithi005 ceph-mon[91801]: osdmap e555: 8 total, 8 up, 8 in 2024-04-26T04:55:10.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:10 smithi005 ceph-mon[95809]: osdmap e555: 8 total, 8 up, 8 in 2024-04-26T04:55:10.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:55:10.360+0000 7f15dde08700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:55:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:10 smithi111 ceph-mon[86360]: osdmap e555: 8 total, 8 up, 8 in 2024-04-26T04:55:11.222 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 34 tests from TestInternal (37449 ms total) 2024-04-26T04:55:11.223 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:55:11.223 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 17 tests from TestImageWatcher 2024-04-26T04:55:11.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: pgmap v466: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 167 KiB/s wr, 729 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: osdmap e556: 8 total, 8 up, 8 in 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: pgmap v468: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 142 KiB/s wr, 609 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: pgmap v469: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 418 KiB/s rd, 163 KiB/s wr, 700 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1574594383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: pgmap v466: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 167 KiB/s wr, 729 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: osdmap e556: 8 total, 8 up, 8 in 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: pgmap v468: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 142 KiB/s wr, 609 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: pgmap v469: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 418 KiB/s rd, 163 KiB/s wr, 700 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:55:11.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:11 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1574594383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: pgmap v466: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 167 KiB/s wr, 729 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: osdmap e556: 8 total, 8 up, 8 in 2024-04-26T04:55:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: pgmap v468: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 142 KiB/s wr, 609 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: pgmap v469: 105 pgs: 24 peering, 7 active+undersized+degraded, 12 active+undersized, 62 active+clean; 5.0 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 418 KiB/s rd, 163 KiB/s wr, 700 op/s; 34/1029 objects degraded (3.304%) 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:55:11.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:11 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1574594383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:12.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:12 smithi005 ceph-mon[91801]: osdmap e557: 8 total, 8 up, 8 in 2024-04-26T04:55:12.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:12 smithi005 ceph-mon[95809]: osdmap e557: 8 total, 8 up, 8 in 2024-04-26T04:55:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:12 smithi111 ceph-mon[86360]: osdmap e557: 8 total, 8 up, 8 in 2024-04-26T04:55:13.228 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-04-26T04:55:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:55:13] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[91801]: pgmap v471: 73 pgs: 19 peering, 54 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[91801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1029 objects degraded (3.304%), 7 pgs degraded) 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[91801]: osdmap e558: 8 total, 8 up, 8 in 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3528465362' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2410043815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[95809]: pgmap v471: 73 pgs: 19 peering, 54 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[95809]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1029 objects degraded (3.304%), 7 pgs degraded) 2024-04-26T04:55:13.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[95809]: osdmap e558: 8 total, 8 up, 8 in 2024-04-26T04:55:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3528465362' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:13.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2410043815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:13 smithi111 ceph-mon[86360]: pgmap v471: 73 pgs: 19 peering, 54 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:13 smithi111 ceph-mon[86360]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1029 objects degraded (3.304%), 7 pgs degraded) 2024-04-26T04:55:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:13 smithi111 ceph-mon[86360]: osdmap e558: 8 total, 8 up, 8 in 2024-04-26T04:55:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3528465362' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:13 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:13.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2410043815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:14 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:14 smithi111 ceph-mon[86360]: osdmap e559: 8 total, 8 up, 8 in 2024-04-26T04:55:14.620 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:14 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:14.621 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:14 smithi005 ceph-mon[91801]: osdmap e559: 8 total, 8 up, 8 in 2024-04-26T04:55:14.621 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:14 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:14.621 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:14 smithi005 ceph-mon[95809]: osdmap e559: 8 total, 8 up, 8 in 2024-04-26T04:55:15.264 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2036 ms) 2024-04-26T04:55:15.264 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-04-26T04:55:15.287 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyFlatten (23 ms) 2024-04-26T04:55:15.287 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-04-26T04:55:15.309 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyResize (22 ms) 2024-04-26T04:55:15.309 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-04-26T04:55:15.331 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (22 ms) 2024-04-26T04:55:15.331 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-04-26T04:55:15.351 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (20 ms) 2024-04-26T04:55:15.351 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-04-26T04:55:15.371 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (20 ms) 2024-04-26T04:55:15.371 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-04-26T04:55:15.393 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (22 ms) 2024-04-26T04:55:15.393 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-04-26T04:55:15.413 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapRename (20 ms) 2024-04-26T04:55:15.413 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-04-26T04:55:15.432 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (19 ms) 2024-04-26T04:55:15.432 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-04-26T04:55:15.452 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (20 ms) 2024-04-26T04:55:15.452 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-04-26T04:55:15.471 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (19 ms) 2024-04-26T04:55:15.471 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-04-26T04:55:15.489 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (18 ms) 2024-04-26T04:55:15.490 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-04-26T04:55:15.509 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyRename (19 ms) 2024-04-26T04:55:15.509 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-04-26T04:55:15.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:15 smithi111 ceph-mon[86360]: pgmap v474: 105 pgs: 21 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:15 smithi111 ceph-mon[86360]: osdmap e560: 8 total, 8 up, 8 in 2024-04-26T04:55:15.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:15 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3844578223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:15.527 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (19 ms) 2024-04-26T04:55:15.527 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-04-26T04:55:15.546 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (19 ms) 2024-04-26T04:55:15.546 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-04-26T04:55:15.566 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (20 ms) 2024-04-26T04:55:15.566 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-04-26T04:55:15.584 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (18 ms) 2024-04-26T04:55:15.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:15 smithi005 ceph-mon[91801]: pgmap v474: 105 pgs: 21 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:15.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:15 smithi005 ceph-mon[91801]: osdmap e560: 8 total, 8 up, 8 in 2024-04-26T04:55:15.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:15 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3844578223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:15.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:15 smithi005 ceph-mon[95809]: pgmap v474: 105 pgs: 21 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:15.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:15 smithi005 ceph-mon[95809]: osdmap e560: 8 total, 8 up, 8 in 2024-04-26T04:55:15.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:15 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3844578223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:16.251 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 17 tests from TestImageWatcher (2356 ms total) 2024-04-26T04:55:16.251 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:55:16.251 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 159 tests from TestLibRBD 2024-04-26T04:55:16.283 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-04-26T04:55:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:16 smithi111 ceph-mon[86360]: osdmap e561: 8 total, 8 up, 8 in 2024-04-26T04:55:16.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:16 smithi005 ceph-mon[95809]: osdmap e561: 8 total, 8 up, 8 in 2024-04-26T04:55:16.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:16 smithi005 ceph-mon[91801]: osdmap e561: 8 total, 8 up, 8 in 2024-04-26T04:55:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:17 smithi111 ceph-mon[86360]: pgmap v477: 105 pgs: 21 creating+peering, 84 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 9.0 KiB/s wr, 78 op/s 2024-04-26T04:55:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:17 smithi111 ceph-mon[86360]: osdmap e562: 8 total, 8 up, 8 in 2024-04-26T04:55:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:17 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:17.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[95809]: pgmap v477: 105 pgs: 21 creating+peering, 84 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 9.0 KiB/s wr, 78 op/s 2024-04-26T04:55:17.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[95809]: osdmap e562: 8 total, 8 up, 8 in 2024-04-26T04:55:17.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:17.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:17.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[91801]: pgmap v477: 105 pgs: 21 creating+peering, 84 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 9.0 KiB/s wr, 78 op/s 2024-04-26T04:55:17.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[91801]: osdmap e562: 8 total, 8 up, 8 in 2024-04-26T04:55:17.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:17.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:18.264 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:18 smithi111 ceph-mon[86360]: osdmap e563: 8 total, 8 up, 8 in 2024-04-26T04:55:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:18 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2298841249' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:18 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1109926158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:18.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:18 smithi005 ceph-mon[91801]: osdmap e563: 8 total, 8 up, 8 in 2024-04-26T04:55:18.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:18 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2298841249' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:18.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:18 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1109926158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:18.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:18 smithi005 ceph-mon[95809]: osdmap e563: 8 total, 8 up, 8 in 2024-04-26T04:55:18.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:18 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2298841249' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:18.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:18 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1109926158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:19 smithi111 ceph-mon[86360]: pgmap v480: 105 pgs: 26 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:19 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:19 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2298841249' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:19.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:19 smithi111 ceph-mon[86360]: osdmap e564: 8 total, 8 up, 8 in 2024-04-26T04:55:19.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[91801]: pgmap v480: 105 pgs: 26 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:19.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:19.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2298841249' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:19.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[91801]: osdmap e564: 8 total, 8 up, 8 in 2024-04-26T04:55:19.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[95809]: pgmap v480: 105 pgs: 26 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:19.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:19.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2298841249' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:19.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:19 smithi005 ceph-mon[95809]: osdmap e564: 8 total, 8 up, 8 in 2024-04-26T04:55:20.297 INFO:tasks.workunit.client.0.smithi005.stdout:image has size 2097152 and order 22 2024-04-26T04:55:20.297 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CreateAndStat (4013 ms) 2024-04-26T04:55:20.297 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-04-26T04:55:20.297 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.313 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (17 ms) 2024-04-26T04:55:20.313 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-04-26T04:55:20.313 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.332 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CreateAndStatPP (19 ms) 2024-04-26T04:55:20.332 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.GetId 2024-04-26T04:55:20.332 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.357 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.GetId (25 ms) 2024-04-26T04:55:20.357 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-04-26T04:55:20.357 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.384 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.GetIdPP (27 ms) 2024-04-26T04:55:20.384 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-04-26T04:55:20.384 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.398 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (14 ms) 2024-04-26T04:55:20.398 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-04-26T04:55:20.398 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.413 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (15 ms) 2024-04-26T04:55:20.413 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-04-26T04:55:20.413 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.429 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (16 ms) 2024-04-26T04:55:20.429 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-04-26T04:55:20.429 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.444 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (15 ms) 2024-04-26T04:55:20.444 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.OpenAio 2024-04-26T04:55:20.444 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.462 INFO:tasks.workunit.client.0.smithi005.stdout:image has size 2097152 and order 22 2024-04-26T04:55:20.463 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.OpenAio (19 ms) 2024-04-26T04:55:20.463 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-04-26T04:55:20.465 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.OpenAioFail (2 ms) 2024-04-26T04:55:20.465 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-04-26T04:55:20.465 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.486 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.OpenAioPP (22 ms) 2024-04-26T04:55:20.487 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-04-26T04:55:20.489 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.OpenAioFailPP (3 ms) 2024-04-26T04:55:20.490 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-04-26T04:55:20.490 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:20 smithi111 ceph-mon[86360]: osdmap e565: 8 total, 8 up, 8 in 2024-04-26T04:55:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:20 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2816281392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:20.523 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ResizeAndStat (33 ms) 2024-04-26T04:55:20.523 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-04-26T04:55:20.523 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.549 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (26 ms) 2024-04-26T04:55:20.549 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-04-26T04:55:20.549 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.580 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (31 ms) 2024-04-26T04:55:20.580 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-04-26T04:55:20.580 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:20.613 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (33 ms) 2024-04-26T04:55:20.613 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-04-26T04:55:20.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:20 smithi005 ceph-mon[91801]: osdmap e565: 8 total, 8 up, 8 in 2024-04-26T04:55:20.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:20 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2816281392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:20.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:20 smithi005 ceph-mon[95809]: osdmap e565: 8 total, 8 up, 8 in 2024-04-26T04:55:20.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:20 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2816281392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:21.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:21 smithi005 ceph-mon[91801]: pgmap v483: 105 pgs: 26 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:21.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:21 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:21.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:21 smithi005 ceph-mon[91801]: osdmap e566: 8 total, 8 up, 8 in 2024-04-26T04:55:21.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:21 smithi005 ceph-mon[95809]: pgmap v483: 105 pgs: 26 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:21.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:21 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:21.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:21 smithi005 ceph-mon[95809]: osdmap e566: 8 total, 8 up, 8 in 2024-04-26T04:55:21.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:21 smithi111 ceph-mon[86360]: pgmap v483: 105 pgs: 26 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:21.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:21 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:21.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:21 smithi111 ceph-mon[86360]: osdmap e566: 8 total, 8 up, 8 in 2024-04-26T04:55:22.293 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:22.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[91801]: osdmap e567: 8 total, 8 up, 8 in 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3710597978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3459199660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[95809]: osdmap e567: 8 total, 8 up, 8 in 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3710597978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:22.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:22 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3459199660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:22 smithi111 ceph-mon[86360]: osdmap e567: 8 total, 8 up, 8 in 2024-04-26T04:55:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:22 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3710597978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:22 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:22 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:22.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:22 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3459199660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:55:23] "GET /metrics HTTP/1.1" 200 34173 "" "Prometheus/2.43.0" 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[91801]: pgmap v486: 137 pgs: 32 unknown, 26 creating+peering, 79 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 3.2 KiB/s wr, 31 op/s 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[91801]: osdmap e568: 8 total, 8 up, 8 in 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[95809]: pgmap v486: 137 pgs: 32 unknown, 26 creating+peering, 79 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 3.2 KiB/s wr, 31 op/s 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[95809]: osdmap e568: 8 total, 8 up, 8 in 2024-04-26T04:55:23.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:23 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:23.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:23 smithi111 ceph-mon[86360]: pgmap v486: 137 pgs: 32 unknown, 26 creating+peering, 79 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 3.2 KiB/s wr, 31 op/s 2024-04-26T04:55:23.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:23 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:23.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:23 smithi111 ceph-mon[86360]: osdmap e568: 8 total, 8 up, 8 in 2024-04-26T04:55:23.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:23 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:24.313 INFO:tasks.workunit.client.0.smithi005.stdout:image: image18 2024-04-26T04:55:24.314 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image18 2024-04-26T04:55:24.314 INFO:tasks.workunit.client.0.smithi005.stdout:found image18 2024-04-26T04:55:24.314 INFO:tasks.workunit.client.0.smithi005.stdout:erased image18 2024-04-26T04:55:24.314 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:image: image18 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:image: image19 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image18 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:found image18 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:erased image18 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image19 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:found image19 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:erased image19 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:image: image19 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image19 2024-04-26T04:55:24.395 INFO:tasks.workunit.client.0.smithi005.stdout:found image19 2024-04-26T04:55:24.396 INFO:tasks.workunit.client.0.smithi005.stdout:erased image19 2024-04-26T04:55:24.396 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3782 ms) 2024-04-26T04:55:24.396 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-04-26T04:55:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:24 smithi005 ceph-mon[95809]: osdmap e569: 8 total, 8 up, 8 in 2024-04-26T04:55:24.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:24 smithi005 ceph-mon[91801]: osdmap e569: 8 total, 8 up, 8 in 2024-04-26T04:55:24.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:24 smithi111 ceph-mon[86360]: osdmap e569: 8 total, 8 up, 8 in 2024-04-26T04:55:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[95809]: pgmap v489: 137 pgs: 7 unknown, 130 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-04-26T04:55:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[95809]: osdmap e570: 8 total, 8 up, 8 in 2024-04-26T04:55:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2006042555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[91801]: pgmap v489: 137 pgs: 7 unknown, 130 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-04-26T04:55:25.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:25.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[91801]: osdmap e570: 8 total, 8 up, 8 in 2024-04-26T04:55:25.731 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2006042555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:25.731 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:55:25.377+0000 7f15dde08700 -1 mgr.server reply reply (11) Resource temporarily unavailable 39 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:25.731 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:25 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:55:25.377+0000 7f15dde08700 -1 mgr.server reply reply (16) Device or resource busy 39 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-04-26T04:55:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:25 smithi111 ceph-mon[86360]: pgmap v489: 137 pgs: 7 unknown, 130 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-04-26T04:55:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:25 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:25 smithi111 ceph-mon[86360]: osdmap e570: 8 total, 8 up, 8 in 2024-04-26T04:55:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:25 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2006042555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:26.318 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:26.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[91801]: osdmap e571: 8 total, 8 up, 8 in 2024-04-26T04:55:26.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1855308690' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[91801]: Upgrade: 39 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[95809]: osdmap e571: 8 total, 8 up, 8 in 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1855308690' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:26 smithi005 ceph-mon[95809]: Upgrade: 39 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:26 smithi111 ceph-mon[86360]: osdmap e571: 8 total, 8 up, 8 in 2024-04-26T04:55:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:26 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1855308690' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:26 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:26 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:26 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:26 smithi111 ceph-mon[86360]: Upgrade: 39 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:27.691 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 1 2024-04-26T04:55:27.691 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:27.691 INFO:tasks.workunit.client.0.smithi005.stdout:image: image20 2024-04-26T04:55:27.691 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image20 2024-04-26T04:55:27.691 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:27.702 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 2 2024-04-26T04:55:27.703 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:55:27.703 INFO:tasks.workunit.client.0.smithi005.stdout:image: image20 2024-04-26T04:55:27.703 INFO:tasks.workunit.client.0.smithi005.stdout:image: image21 2024-04-26T04:55:27.703 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image20 2024-04-26T04:55:27.703 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image21 2024-04-26T04:55:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[91801]: pgmap v492: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 223 op/s 2024-04-26T04:55:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[91801]: osdmap e572: 8 total, 8 up, 8 in 2024-04-26T04:55:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/481038379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[95809]: pgmap v492: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 223 op/s 2024-04-26T04:55:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[95809]: osdmap e572: 8 total, 8 up, 8 in 2024-04-26T04:55:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:27 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/481038379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:27 smithi111 ceph-mon[86360]: pgmap v492: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 223 op/s 2024-04-26T04:55:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:27 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:55:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:27 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:27 smithi111 ceph-mon[86360]: osdmap e572: 8 total, 8 up, 8 in 2024-04-26T04:55:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:27 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/481038379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:27.772 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 1 2024-04-26T04:55:27.772 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:27.772 INFO:tasks.workunit.client.0.smithi005.stdout:image: image21 2024-04-26T04:55:27.772 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image21 2024-04-26T04:55:27.772 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (3377 ms) 2024-04-26T04:55:27.772 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCopy 2024-04-26T04:55:28.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:28 smithi005 ceph-mon[91801]: osdmap e573: 8 total, 8 up, 8 in 2024-04-26T04:55:28.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:28 smithi005 ceph-mon[91801]: osdmap e574: 8 total, 8 up, 8 in 2024-04-26T04:55:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:28 smithi005 ceph-mon[95809]: osdmap e573: 8 total, 8 up, 8 in 2024-04-26T04:55:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:28 smithi005 ceph-mon[95809]: osdmap e574: 8 total, 8 up, 8 in 2024-04-26T04:55:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:28 smithi111 ceph-mon[86360]: osdmap e573: 8 total, 8 up, 8 in 2024-04-26T04:55:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:28 smithi111 ceph-mon[86360]: osdmap e574: 8 total, 8 up, 8 in 2024-04-26T04:55:29.688 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:29.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[91801]: pgmap v496: 169 pgs: 6 creating+peering, 163 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 3.6 KiB/s wr, 42 op/s 2024-04-26T04:55:29.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[91801]: osdmap e575: 8 total, 8 up, 8 in 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1591895625' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3923315233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[95809]: pgmap v496: 169 pgs: 6 creating+peering, 163 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 3.6 KiB/s wr, 42 op/s 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[95809]: osdmap e575: 8 total, 8 up, 8 in 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1591895625' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:29 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3923315233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:29 smithi111 ceph-mon[86360]: pgmap v496: 169 pgs: 6 creating+peering, 163 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 3.6 KiB/s wr, 42 op/s 2024-04-26T04:55:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:29 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:55:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:29 smithi111 ceph-mon[86360]: osdmap e575: 8 total, 8 up, 8 in 2024-04-26T04:55:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:29 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1591895625' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:29 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:29 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3923315233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:30.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:30 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:30.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:30 smithi005 ceph-mon[95809]: osdmap e576: 8 total, 8 up, 8 in 2024-04-26T04:55:30.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:30 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:30.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:30 smithi005 ceph-mon[91801]: osdmap e576: 8 total, 8 up, 8 in 2024-04-26T04:55:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:30 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:30 smithi111 ceph-mon[86360]: osdmap e576: 8 total, 8 up, 8 in 2024-04-26T04:55:31.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[95809]: pgmap v499: 201 pgs: 32 unknown, 6 creating+peering, 163 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.3 KiB/s wr, 38 op/s 2024-04-26T04:55:31.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[95809]: osdmap e577: 8 total, 8 up, 8 in 2024-04-26T04:55:31.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4079034255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:31.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[91801]: pgmap v499: 201 pgs: 32 unknown, 6 creating+peering, 163 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.3 KiB/s wr, 38 op/s 2024-04-26T04:55:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[91801]: osdmap e577: 8 total, 8 up, 8 in 2024-04-26T04:55:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4079034255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:31 smithi111 ceph-mon[86360]: pgmap v499: 201 pgs: 32 unknown, 6 creating+peering, 163 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.3 KiB/s wr, 38 op/s 2024-04-26T04:55:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:31 smithi111 ceph-mon[86360]: osdmap e577: 8 total, 8 up, 8 in 2024-04-26T04:55:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:31 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4079034255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:image: image22 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image22 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:found image22 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:erased image22 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:image: image22 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:image: image23 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image22 2024-04-26T04:55:32.036 INFO:tasks.workunit.client.0.smithi005.stdout:found image22 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:erased image22 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:found image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:erased image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:image: image22 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:image: image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:image: image24 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image22 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:found image22 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:erased image22 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:found image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:erased image23 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image24 2024-04-26T04:55:32.037 INFO:tasks.workunit.client.0.smithi005.stdout:found image24 2024-04-26T04:55:32.038 INFO:tasks.workunit.client.0.smithi005.stdout:erased image24 2024-04-26T04:55:32.038 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCopy (4264 ms) 2024-04-26T04:55:32.038 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-04-26T04:55:32.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[95809]: osdmap e578: 8 total, 8 up, 8 in 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[95809]: osdmap e579: 8 total, 8 up, 8 in 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2388794855' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[91801]: osdmap e578: 8 total, 8 up, 8 in 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[91801]: osdmap e579: 8 total, 8 up, 8 in 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2388794855' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:32.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:32 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:32 smithi111 ceph-mon[86360]: osdmap e578: 8 total, 8 up, 8 in 2024-04-26T04:55:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:32 smithi111 ceph-mon[86360]: osdmap e579: 8 total, 8 up, 8 in 2024-04-26T04:55:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:32 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2388794855' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:32 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:55:33] "GET /metrics HTTP/1.1" 200 43733 "" "Prometheus/2.43.0" 2024-04-26T04:55:33.675 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:33.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-mon[91801]: pgmap v502: 201 pgs: 23 unknown, 178 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-04-26T04:55:33.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:33.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-mon[91801]: osdmap e580: 8 total, 8 up, 8 in 2024-04-26T04:55:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-mon[95809]: pgmap v502: 201 pgs: 23 unknown, 178 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-04-26T04:55:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:33 smithi005 ceph-mon[95809]: osdmap e580: 8 total, 8 up, 8 in 2024-04-26T04:55:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:33 smithi111 ceph-mon[86360]: pgmap v502: 201 pgs: 23 unknown, 178 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-04-26T04:55:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:33 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:33 smithi111 ceph-mon[86360]: osdmap e580: 8 total, 8 up, 8 in 2024-04-26T04:55:34.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:34 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3071471741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:34.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:34 smithi005 ceph-mon[91801]: osdmap e581: 8 total, 8 up, 8 in 2024-04-26T04:55:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:34 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3071471741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:34 smithi005 ceph-mon[95809]: osdmap e581: 8 total, 8 up, 8 in 2024-04-26T04:55:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:34 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3071471741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:34 smithi111 ceph-mon[86360]: osdmap e581: 8 total, 8 up, 8 in 2024-04-26T04:55:35.950 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 1 2024-04-26T04:55:35.950 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:35.950 INFO:tasks.workunit.client.0.smithi005.stdout:image: image25 2024-04-26T04:55:35.950 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image25 2024-04-26T04:55:35.969 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 2 2024-04-26T04:55:35.969 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:55:35.969 INFO:tasks.workunit.client.0.smithi005.stdout:image: image25 2024-04-26T04:55:35.969 INFO:tasks.workunit.client.0.smithi005.stdout:image: image26 2024-04-26T04:55:35.969 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image25 2024-04-26T04:55:35.969 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image26 2024-04-26T04:55:35.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:35 smithi005 ceph-mon[91801]: pgmap v505: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 317 KiB/s rd, 23 KiB/s wr, 382 op/s 2024-04-26T04:55:35.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:35 smithi005 ceph-mon[91801]: osdmap e582: 8 total, 8 up, 8 in 2024-04-26T04:55:35.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:35 smithi005 ceph-mon[95809]: pgmap v505: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 317 KiB/s rd, 23 KiB/s wr, 382 op/s 2024-04-26T04:55:35.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:35 smithi005 ceph-mon[95809]: osdmap e582: 8 total, 8 up, 8 in 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 3 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:image: image25 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:image: image26 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:image: image27 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image25 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image26 2024-04-26T04:55:36.016 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image27 2024-04-26T04:55:36.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:35 smithi111 ceph-mon[86360]: pgmap v505: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 317 KiB/s rd, 23 KiB/s wr, 382 op/s 2024-04-26T04:55:36.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:35 smithi111 ceph-mon[86360]: osdmap e582: 8 total, 8 up, 8 in 2024-04-26T04:55:36.048 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCopyPP (4013 ms) 2024-04-26T04:55:36.049 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-04-26T04:55:36.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:36 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4292006540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:36.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:36 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4292006540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:36 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4292006540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:37.677 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:37.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[91801]: pgmap v508: 233 pgs: 20 unknown, 213 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 483 KiB/s rd, 32 KiB/s wr, 574 op/s 2024-04-26T04:55:37.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[91801]: osdmap e583: 8 total, 8 up, 8 in 2024-04-26T04:55:37.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3629462029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[91801]: osdmap e584: 8 total, 8 up, 8 in 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[95809]: pgmap v508: 233 pgs: 20 unknown, 213 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 483 KiB/s rd, 32 KiB/s wr, 574 op/s 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[95809]: osdmap e583: 8 total, 8 up, 8 in 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3629462029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:37.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:37 smithi005 ceph-mon[95809]: osdmap e584: 8 total, 8 up, 8 in 2024-04-26T04:55:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:37 smithi111 ceph-mon[86360]: pgmap v508: 233 pgs: 20 unknown, 213 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 483 KiB/s rd, 32 KiB/s wr, 574 op/s 2024-04-26T04:55:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:37 smithi111 ceph-mon[86360]: osdmap e583: 8 total, 8 up, 8 in 2024-04-26T04:55:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:37 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3629462029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:37 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:37 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:38.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:37 smithi111 ceph-mon[86360]: osdmap e584: 8 total, 8 up, 8 in 2024-04-26T04:55:38.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:38 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/369590299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:38.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:38 smithi005 ceph-mon[91801]: osdmap e585: 8 total, 8 up, 8 in 2024-04-26T04:55:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:38 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/369590299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:38.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:38 smithi005 ceph-mon[95809]: osdmap e585: 8 total, 8 up, 8 in 2024-04-26T04:55:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:38 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/369590299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:38 smithi111 ceph-mon[86360]: osdmap e585: 8 total, 8 up, 8 in 2024-04-26T04:55:39.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:39 smithi005 ceph-mon[95809]: pgmap v511: 265 pgs: 24 creating+peering, 8 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 12 KiB/s wr, 209 op/s 2024-04-26T04:55:39.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:39 smithi005 ceph-mon[95809]: osdmap e586: 8 total, 8 up, 8 in 2024-04-26T04:55:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:39 smithi005 ceph-mon[91801]: pgmap v511: 265 pgs: 24 creating+peering, 8 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 12 KiB/s wr, 209 op/s 2024-04-26T04:55:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:39 smithi005 ceph-mon[91801]: osdmap e586: 8 total, 8 up, 8 in 2024-04-26T04:55:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:39 smithi111 ceph-mon[86360]: pgmap v511: 265 pgs: 24 creating+peering, 8 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 12 KiB/s wr, 209 op/s 2024-04-26T04:55:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:39 smithi111 ceph-mon[86360]: osdmap e586: 8 total, 8 up, 8 in 2024-04-26T04:55:40.721 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:40 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:55:40.385+0000 7f15dde08700 -1 mgr.server reply reply (11) Resource temporarily unavailable 8 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:40.721 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:40 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:55:40.385+0000 7f15dde08700 -1 mgr.server reply reply (16) Device or resource busy 8 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T04:55:40.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:40 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:40 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:40 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[91801]: pgmap v514: 265 pgs: 24 creating+peering, 8 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[91801]: Upgrade: 8 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[91801]: osdmap e587: 8 total, 8 up, 8 in 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/82794757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[95809]: pgmap v514: 265 pgs: 24 creating+peering, 8 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[95809]: Upgrade: 8 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[95809]: osdmap e587: 8 total, 8 up, 8 in 2024-04-26T04:55:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:41 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/82794757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:41 smithi111 ceph-mon[86360]: pgmap v514: 265 pgs: 24 creating+peering, 8 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-04-26T04:55:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:41 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:41 smithi111 ceph-mon[86360]: Upgrade: 8 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:55:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:41 smithi111 ceph-mon[86360]: osdmap e587: 8 total, 8 up, 8 in 2024-04-26T04:55:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:41 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/82794757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:42 smithi111 ceph-mon[86360]: osdmap e588: 8 total, 8 up, 8 in 2024-04-26T04:55:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:42 smithi111 ceph-mon[86360]: osdmap e589: 8 total, 8 up, 8 in 2024-04-26T04:55:43.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:42 smithi005 ceph-mon[91801]: osdmap e588: 8 total, 8 up, 8 in 2024-04-26T04:55:43.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:42 smithi005 ceph-mon[91801]: osdmap e589: 8 total, 8 up, 8 in 2024-04-26T04:55:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:42 smithi005 ceph-mon[95809]: osdmap e588: 8 total, 8 up, 8 in 2024-04-26T04:55:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:42 smithi005 ceph-mon[95809]: osdmap e589: 8 total, 8 up, 8 in 2024-04-26T04:55:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:55:43] "GET /metrics HTTP/1.1" 200 43733 "" "Prometheus/2.43.0" 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:image: image28 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image28 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:found image28 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:erased image28 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:image: image28 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:image: image29 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image28 2024-04-26T04:55:43.813 INFO:tasks.workunit.client.0.smithi005.stdout:found image28 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:erased image28 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image29 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:found image29 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:erased image29 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:image: image28 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:image: image29 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:image: image30 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image28 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:found image28 2024-04-26T04:55:43.814 INFO:tasks.workunit.client.0.smithi005.stdout:erased image28 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image29 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:found image29 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:erased image29 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image30 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:found image30 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:erased image30 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:image: image28 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:image: image29 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:image: image30 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:image: image31 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image28 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:found image28 2024-04-26T04:55:43.815 INFO:tasks.workunit.client.0.smithi005.stdout:erased image28 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image29 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:found image29 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:erased image29 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image30 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:found image30 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:erased image30 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image31 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:found image31 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:erased image31 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:image: image28 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:image: image29 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:image: image30 2024-04-26T04:55:43.816 INFO:tasks.workunit.client.0.smithi005.stdout:image: image31 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:image: image32 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image28 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:found image28 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:erased image28 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image29 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:found image29 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:erased image29 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image30 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:found image30 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:erased image30 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image31 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:found image31 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:erased image31 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image32 2024-04-26T04:55:43.817 INFO:tasks.workunit.client.0.smithi005.stdout:found image32 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:erased image32 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:image: image28 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:image: image29 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:image: image30 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:image: image31 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:image: image32 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:image: image33 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image28 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:found image28 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:erased image28 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image29 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:found image29 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:erased image29 2024-04-26T04:55:43.818 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image30 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:found image30 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:erased image30 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image31 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:found image31 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:erased image31 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image32 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:found image32 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:erased image32 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image33 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:found image33 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:erased image33 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestDeepCopy (7764 ms) 2024-04-26T04:55:43.819 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-04-26T04:55:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:43 smithi111 ceph-mon[86360]: pgmap v517: 265 pgs: 24 creating+peering, 241 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 13 KiB/s wr, 282 op/s 2024-04-26T04:55:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:43 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4135898718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:44.150 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:43 smithi005 ceph-mon[91801]: pgmap v517: 265 pgs: 24 creating+peering, 241 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 13 KiB/s wr, 282 op/s 2024-04-26T04:55:44.150 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:43 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4135898718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:44.150 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:43 smithi005 ceph-mon[95809]: pgmap v517: 265 pgs: 24 creating+peering, 241 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 13 KiB/s wr, 282 op/s 2024-04-26T04:55:44.150 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:43 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4135898718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:44 smithi111 ceph-mon[86360]: osdmap e590: 8 total, 8 up, 8 in 2024-04-26T04:55:45.114 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:44 smithi005 ceph-mon[91801]: osdmap e590: 8 total, 8 up, 8 in 2024-04-26T04:55:45.115 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:44 smithi005 ceph-mon[95809]: osdmap e590: 8 total, 8 up, 8 in 2024-04-26T04:55:45.758 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:45 smithi111 ceph-mon[86360]: pgmap v520: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 655 KiB/s rd, 39 KiB/s wr, 762 op/s 2024-04-26T04:55:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:45 smithi111 ceph-mon[86360]: osdmap e591: 8 total, 8 up, 8 in 2024-04-26T04:55:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:45 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4137914020' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:45 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:45 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2418127810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[91801]: pgmap v520: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 655 KiB/s rd, 39 KiB/s wr, 762 op/s 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[91801]: osdmap e591: 8 total, 8 up, 8 in 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4137914020' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2418127810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[95809]: pgmap v520: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 655 KiB/s rd, 39 KiB/s wr, 762 op/s 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[95809]: osdmap e591: 8 total, 8 up, 8 in 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4137914020' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:55:46.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:45 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2418127810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:46 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:46 smithi111 ceph-mon[86360]: osdmap e592: 8 total, 8 up, 8 in 2024-04-26T04:55:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:46 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:46 smithi005 ceph-mon[91801]: osdmap e592: 8 total, 8 up, 8 in 2024-04-26T04:55:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:46 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:55:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:46 smithi005 ceph-mon[95809]: osdmap e592: 8 total, 8 up, 8 in 2024-04-26T04:55:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:47.939 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 1 2024-04-26T04:55:47.939 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:47.939 INFO:tasks.workunit.client.0.smithi005.stdout:image: image34 2024-04-26T04:55:47.940 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image34 2024-04-26T04:55:47.964 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 2 2024-04-26T04:55:47.964 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:55:47.964 INFO:tasks.workunit.client.0.smithi005.stdout:image: image34 2024-04-26T04:55:47.964 INFO:tasks.workunit.client.0.smithi005.stdout:image: image35 2024-04-26T04:55:47.964 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image34 2024-04-26T04:55:47.964 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image35 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:num images is: 3 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:image: image34 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:image: image35 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:image: image36 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image34 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image35 2024-04-26T04:55:48.014 INFO:tasks.workunit.client.0.smithi005.stdout:expected = image36 2024-04-26T04:55:48.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:47 smithi111 ceph-mon[86360]: pgmap v523: 297 pgs: 22 unknown, 275 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 29 KiB/s wr, 532 op/s 2024-04-26T04:55:48.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:47 smithi111 ceph-mon[86360]: osdmap e593: 8 total, 8 up, 8 in 2024-04-26T04:55:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:47 smithi111 ceph-mon[86360]: osdmap e594: 8 total, 8 up, 8 in 2024-04-26T04:55:48.045 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4232 ms) 2024-04-26T04:55:48.045 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-04-26T04:55:48.045 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:47 smithi005 ceph-mon[91801]: pgmap v523: 297 pgs: 22 unknown, 275 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 29 KiB/s wr, 532 op/s 2024-04-26T04:55:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:47 smithi005 ceph-mon[91801]: osdmap e593: 8 total, 8 up, 8 in 2024-04-26T04:55:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:47 smithi005 ceph-mon[91801]: osdmap e594: 8 total, 8 up, 8 in 2024-04-26T04:55:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:47 smithi005 ceph-mon[95809]: pgmap v523: 297 pgs: 22 unknown, 275 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 445 KiB/s rd, 29 KiB/s wr, 532 op/s 2024-04-26T04:55:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:47 smithi005 ceph-mon[95809]: osdmap e593: 8 total, 8 up, 8 in 2024-04-26T04:55:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:47 smithi005 ceph-mon[95809]: osdmap e594: 8 total, 8 up, 8 in 2024-04-26T04:55:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:48 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3038537840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:49.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:48 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3038537840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:48 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3038537840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:50.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:49 smithi005 ceph-mon[91801]: pgmap v526: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 4.7 KiB/s wr, 134 op/s 2024-04-26T04:55:50.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:49 smithi005 ceph-mon[91801]: osdmap e595: 8 total, 8 up, 8 in 2024-04-26T04:55:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:49 smithi005 ceph-mon[95809]: pgmap v526: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 4.7 KiB/s wr, 134 op/s 2024-04-26T04:55:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:49 smithi005 ceph-mon[95809]: osdmap e595: 8 total, 8 up, 8 in 2024-04-26T04:55:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:49 smithi111 ceph-mon[86360]: pgmap v526: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 4.7 KiB/s wr, 134 op/s 2024-04-26T04:55:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:49 smithi111 ceph-mon[86360]: osdmap e595: 8 total, 8 up, 8 in 2024-04-26T04:55:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:50 smithi005 ceph-mon[91801]: osdmap e596: 8 total, 8 up, 8 in 2024-04-26T04:55:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:50 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/594855062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:50 smithi005 ceph-mon[95809]: osdmap e596: 8 total, 8 up, 8 in 2024-04-26T04:55:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:50 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/594855062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:50 smithi111 ceph-mon[86360]: osdmap e596: 8 total, 8 up, 8 in 2024-04-26T04:55:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:50 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/594855062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1 with size 2097152 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 2 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1 with size 2097152 2024-04-26T04:55:51.801 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2 with size 4194304 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2 with size 4194304 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3756 ms) 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-04-26T04:55:51.802 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:51 smithi005 ceph-mon[91801]: pgmap v529: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 2.0 KiB/s wr, 81 op/s 2024-04-26T04:55:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:51 smithi005 ceph-mon[91801]: osdmap e597: 8 total, 8 up, 8 in 2024-04-26T04:55:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:51 smithi005 ceph-mon[95809]: pgmap v529: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 2.0 KiB/s wr, 81 op/s 2024-04-26T04:55:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:51 smithi005 ceph-mon[95809]: osdmap e597: 8 total, 8 up, 8 in 2024-04-26T04:55:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:51 smithi111 ceph-mon[86360]: pgmap v529: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 2.0 KiB/s wr, 81 op/s 2024-04-26T04:55:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:51 smithi111 ceph-mon[86360]: osdmap e597: 8 total, 8 up, 8 in 2024-04-26T04:55:53.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:52 smithi005 ceph-mon[91801]: osdmap e598: 8 total, 8 up, 8 in 2024-04-26T04:55:53.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:52 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3449166730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:53.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:52 smithi005 ceph-mon[95809]: osdmap e598: 8 total, 8 up, 8 in 2024-04-26T04:55:53.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:52 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3449166730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:52 smithi111 ceph-mon[86360]: osdmap e598: 8 total, 8 up, 8 in 2024-04-26T04:55:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:52 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3449166730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:55:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:55:53] "GET /metrics HTTP/1.1" 200 50972 "" "Prometheus/2.43.0" 2024-04-26T04:55:53.822 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2021 ms) 2024-04-26T04:55:53.822 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-04-26T04:55:53.822 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:54.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:53 smithi005 ceph-mon[91801]: pgmap v532: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 2.2 KiB/s wr, 59 op/s 2024-04-26T04:55:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:53 smithi005 ceph-mon[91801]: osdmap e599: 8 total, 8 up, 8 in 2024-04-26T04:55:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:53 smithi005 ceph-mon[95809]: pgmap v532: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 2.2 KiB/s wr, 59 op/s 2024-04-26T04:55:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:53 smithi005 ceph-mon[95809]: osdmap e599: 8 total, 8 up, 8 in 2024-04-26T04:55:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:53 smithi111 ceph-mon[86360]: pgmap v532: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 2.2 KiB/s wr, 59 op/s 2024-04-26T04:55:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:53 smithi111 ceph-mon[86360]: osdmap e599: 8 total, 8 up, 8 in 2024-04-26T04:55:54.822 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:55:54.822 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:54.822 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:55:54.822 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1 with size 2097152 2024-04-26T04:55:54.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:54 smithi005 ceph-mon[95809]: osdmap e600: 8 total, 8 up, 8 in 2024-04-26T04:55:54.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:54 smithi005 ceph-mon[91801]: osdmap e600: 8 total, 8 up, 8 in 2024-04-26T04:55:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:54 smithi111 ceph-mon[86360]: osdmap e600: 8 total, 8 up, 8 in 2024-04-26T04:55:55.825 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 2 2024-04-26T04:55:55.825 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:55:55.825 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:55:55.825 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:55:55.825 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1 with size 2097152 2024-04-26T04:55:55.825 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2 with size 4194304 2024-04-26T04:55:56.113 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[95809]: pgmap v535: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-04-26T04:55:56.113 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[95809]: osdmap e601: 8 total, 8 up, 8 in 2024-04-26T04:55:56.114 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/330632811' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:56.114 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:56.114 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[91801]: pgmap v535: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-04-26T04:55:56.114 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[91801]: osdmap e601: 8 total, 8 up, 8 in 2024-04-26T04:55:56.114 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/330632811' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:56.114 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:55 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:55 smithi111 ceph-mon[86360]: pgmap v535: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-04-26T04:55:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:55 smithi111 ceph-mon[86360]: osdmap e601: 8 total, 8 up, 8 in 2024-04-26T04:55:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:55 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/330632811' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:55 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:56.824 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:55:56.824 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:55:56.824 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:55:56.824 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2 with size 4194304 2024-04-26T04:55:57.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:57.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[95809]: Upgrade: osd.2 is safe to restart 2024-04-26T04:55:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[95809]: osdmap e602: 8 total, 8 up, 8 in 2024-04-26T04:55:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[91801]: Upgrade: osd.2 is safe to restart 2024-04-26T04:55:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[91801]: osdmap e602: 8 total, 8 up, 8 in 2024-04-26T04:55:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:56 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:56 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:56 smithi111 ceph-mon[86360]: Upgrade: osd.2 is safe to restart 2024-04-26T04:55:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:56 smithi111 ceph-mon[86360]: osdmap e602: 8 total, 8 up, 8 in 2024-04-26T04:55:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:56 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:57.682 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:55:57.682 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:55:57.689 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (3867 ms) 2024-04-26T04:55:57.689 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-04-26T04:55:57.689 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:55:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: pgmap v538: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:55:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: osdmap e603: 8 total, 8 up, 8 in 2024-04-26T04:55:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: Upgrade: Updating osd.2 2024-04-26T04:55:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:55:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: Deploying daemon osd.2 on smithi005 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2477892629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[91801]: osdmap e604: 8 total, 8 up, 8 in 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: pgmap v538: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: osdmap e603: 8 total, 8 up, 8 in 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: Upgrade: Updating osd.2 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: Deploying daemon osd.2 on smithi005 2024-04-26T04:55:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2477892629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:57.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:57 smithi005 ceph-mon[95809]: osdmap e604: 8 total, 8 up, 8 in 2024-04-26T04:55:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: pgmap v538: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 26 KiB/s wr, 419 op/s 2024-04-26T04:55:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: osdmap e603: 8 total, 8 up, 8 in 2024-04-26T04:55:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: Upgrade: Updating osd.2 2024-04-26T04:55:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:55:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:55:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:58.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: Deploying daemon osd.2 on smithi005 2024-04-26T04:55:58.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2477892629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:58.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:57 smithi111 ceph-mon[86360]: osdmap e604: 8 total, 8 up, 8 in 2024-04-26T04:55:59.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:59 smithi005 ceph-mon[91801]: pgmap v541: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-04-26T04:55:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:55:59 smithi005 ceph-mon[91801]: osdmap e605: 8 total, 8 up, 8 in 2024-04-26T04:55:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:59 smithi005 ceph-mon[95809]: pgmap v541: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-04-26T04:55:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:55:59 smithi005 ceph-mon[95809]: osdmap e605: 8 total, 8 up, 8 in 2024-04-26T04:56:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:59 smithi111 ceph-mon[86360]: pgmap v541: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-04-26T04:56:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:55:59 smithi111 ceph-mon[86360]: osdmap e605: 8 total, 8 up, 8 in 2024-04-26T04:56:00.684 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:00 smithi005 systemd[1]: Stopping Ceph osd.2 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:56:00.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-mon[95809]: osdmap e606: 8 total, 8 up, 8 in 2024-04-26T04:56:00.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3763913578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-mon[91801]: osdmap e606: 8 total, 8 up, 8 in 2024-04-26T04:56:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3763913578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:00.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[52705]: 2024-04-26T04:56:00.697+0000 7f2571012700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:56:00.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[52705]: 2024-04-26T04:56:00.697+0000 7f2571012700 -1 osd.2 606 *** Got signal Terminated *** 2024-04-26T04:56:00.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[52705]: 2024-04-26T04:56:00.697+0000 7f2571012700 -1 osd.2 606 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:56:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:00 smithi111 ceph-mon[86360]: osdmap e606: 8 total, 8 up, 8 in 2024-04-26T04:56:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:00 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3763913578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: pgmap v544: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: osdmap e607: 8 total, 8 up, 8 in 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: osd.2 marked itself down and dead 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]: dispatch 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-04-26T04:56:01.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: pgmap v544: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: osdmap e607: 8 total, 8 up, 8 in 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: osd.2 marked itself down and dead 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]: dispatch 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-04-26T04:56:01.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: pgmap v544: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-04-26T04:56:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: osdmap e607: 8 total, 8 up, 8 in 2024-04-26T04:56:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: osd.2 marked itself down and dead 2024-04-26T04:56:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-04-26T04:56:02.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-04-26T04:56:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]: dispatch 2024-04-26T04:56:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-04-26T04:56:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]: dispatch 2024-04-26T04:56:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-04-26T04:56:02.020 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:02.522 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:02 smithi005 podman[114832]: 2024-04-26 04:56:02.243082744 +0000 UTC m=+1.656160711 container died f25e1cd874cad528ae3630d92d847dcd20ccb5f131387b4733d9d247f175ee1d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:56:02.889 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:02 smithi005 podman[114832]: 2024-04-26 04:56:02.620884321 +0000 UTC m=+2.033962296 container cleanup f25e1cd874cad528ae3630d92d847dcd20ccb5f131387b4733d9d247f175ee1d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T04:56:02.890 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:02 smithi005 bash[114832]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]': finished 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]': finished 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]': finished 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-04-26T04:56:02.890 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: osdmap e608: 8 total, 7 up, 8 in 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2471284633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[95809]: osdmap e609: 8 total, 7 up, 8 in 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]': finished 2024-04-26T04:56:02.891 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]': finished 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]': finished 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]': finished 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: osdmap e608: 8 total, 7 up, 8 in 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2471284633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:02.892 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:02 smithi005 ceph-mon[91801]: osdmap e609: 8 total, 7 up, 8 in 2024-04-26T04:56:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:56:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 0]}]': finished 2024-04-26T04:56:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-04-26T04:56:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 5]}]': finished 2024-04-26T04:56:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]': finished 2024-04-26T04:56:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.d", "id": [7, 0]}]': finished 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.17", "id": [0, 2]}]': finished 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: osdmap e608: 8 total, 7 up, 8 in 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2471284633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:03.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:02 smithi111 ceph-mon[86360]: osdmap e609: 8 total, 7 up, 8 in 2024-04-26T04:56:03.195 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:02 smithi005 podman[114888]: 2024-04-26 04:56:02.88932115 +0000 UTC m=+0.644442524 container remove f25e1cd874cad528ae3630d92d847dcd20ccb5f131387b4733d9d247f175ee1d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5) 2024-04-26T04:56:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:56:03] "GET /metrics HTTP/1.1" 200 51012 "" "Prometheus/2.43.0" 2024-04-26T04:56:03.480 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:03 smithi005 podman[114975]: 2024-04-26 04:56:03.129290555 +0000 UTC m=+0.018620674 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:03.480 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:03 smithi005 podman[114975]: 2024-04-26 04:56:03.256925785 +0000 UTC m=+0.146255891 container create de64214f4d64cf7da81a1ed8c51fbaed6d3e8f0186f2a757093693bddfa5f7a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:56:03.480 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:03 smithi005 podman[114975]: 2024-04-26 04:56:03.426651064 +0000 UTC m=+0.315981171 container init de64214f4d64cf7da81a1ed8c51fbaed6d3e8f0186f2a757093693bddfa5f7a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T04:56:03.480 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:03 smithi005 podman[114975]: 2024-04-26 04:56:03.429896471 +0000 UTC m=+0.319226579 container start de64214f4d64cf7da81a1ed8c51fbaed6d3e8f0186f2a757093693bddfa5f7a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:56:03.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:03 smithi005 ceph-mon[91801]: pgmap v547: 297 pgs: 28 stale+active+clean, 269 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:03.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:03 smithi005 ceph-mon[91801]: osdmap e610: 8 total, 7 up, 8 in 2024-04-26T04:56:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:03 smithi005 ceph-mon[95809]: pgmap v547: 297 pgs: 28 stale+active+clean, 269 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:03 smithi005 ceph-mon[95809]: osdmap e610: 8 total, 7 up, 8 in 2024-04-26T04:56:03.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:03 smithi005 podman[114975]: 2024-04-26 04:56:03.484271524 +0000 UTC m=+0.373601644 container attach de64214f4d64cf7da81a1ed8c51fbaed6d3e8f0186f2a757093693bddfa5f7a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-26T04:56:03.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:03 smithi005 podman[114975]: 2024-04-26 04:56:03.734018072 +0000 UTC m=+0.623348190 container died de64214f4d64cf7da81a1ed8c51fbaed6d3e8f0186f2a757093693bddfa5f7a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T04:56:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:03 smithi111 ceph-mon[86360]: pgmap v547: 297 pgs: 28 stale+active+clean, 269 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:03 smithi111 ceph-mon[86360]: osdmap e610: 8 total, 7 up, 8 in 2024-04-26T04:56:04.702 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 podman[115001]: 2024-04-26 04:56:04.263169527 +0000 UTC m=+0.519855027 container remove de64214f4d64cf7da81a1ed8c51fbaed6d3e8f0186f2a757093693bddfa5f7a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T04:56:04.702 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T04:56:04.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 systemd[1]: Stopped Ceph osd.2 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:56:04.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.2.service: Consumed 26.858s CPU time. 2024-04-26T04:56:04.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 systemd[1]: Starting Ceph osd.2 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:56:04.716 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:56:04.716 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:56:04.725 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (7036 ms) 2024-04-26T04:56:04.725 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-04-26T04:56:04.725 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:04.953 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:04 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/524436517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:04.954 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:04 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/524436517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:04.954 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 podman[115134]: 2024-04-26 04:56:04.630774462 +0000 UTC m=+0.018902194 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:04.955 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 podman[115134]: 2024-04-26 04:56:04.728099315 +0000 UTC m=+0.116227077 container create b6928f8fdfd418064e79d5295c46736c259447daad4febcc0db78e7dfdd52e1c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:56:04.955 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 podman[115134]: 2024-04-26 04:56:04.950118974 +0000 UTC m=+0.338246692 container init b6928f8fdfd418064e79d5295c46736c259447daad4febcc0db78e7dfdd52e1c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-26T04:56:04.955 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:04 smithi005 podman[115134]: 2024-04-26 04:56:04.953310733 +0000 UTC m=+0.341438451 container start b6928f8fdfd418064e79d5295c46736c259447daad4febcc0db78e7dfdd52e1c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T04:56:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:04 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/524436517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:05.229 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 podman[115134]: 2024-04-26 04:56:05.050387613 +0000 UTC m=+0.438515343 container attach b6928f8fdfd418064e79d5295c46736c259447daad4febcc0db78e7dfdd52e1c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T04:56:05.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:56:05.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T04:56:05.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:56:05.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T04:56:05.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T04:56:05.703 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 bash[115134]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate[115149]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T04:56:05.704 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:05 smithi005 podman[115134]: 2024-04-26 04:56:05.437142433 +0000 UTC m=+0.825270147 container died b6928f8fdfd418064e79d5295c46736c259447daad4febcc0db78e7dfdd52e1c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-26T04:56:05.732 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:56:05.732 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:56:05.732 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:56:05.732 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1 with size 2097152 2024-04-26T04:56:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-mon[95809]: pgmap v550: 297 pgs: 73 peering, 10 stale+active+clean, 214 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 4.7 KiB/s wr, 61 op/s 2024-04-26T04:56:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-mon[95809]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-26T04:56:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-mon[95809]: osdmap e611: 8 total, 7 up, 8 in 2024-04-26T04:56:05.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-mon[91801]: pgmap v550: 297 pgs: 73 peering, 10 stale+active+clean, 214 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 4.7 KiB/s wr, 61 op/s 2024-04-26T04:56:05.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-mon[91801]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-26T04:56:05.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:05 smithi005 ceph-mon[91801]: osdmap e611: 8 total, 7 up, 8 in 2024-04-26T04:56:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:05 smithi111 ceph-mon[86360]: pgmap v550: 297 pgs: 73 peering, 10 stale+active+clean, 214 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 4.7 KiB/s wr, 61 op/s 2024-04-26T04:56:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:05 smithi111 ceph-mon[86360]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-26T04:56:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:05 smithi111 ceph-mon[86360]: osdmap e611: 8 total, 7 up, 8 in 2024-04-26T04:56:06.336 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 podman[115266]: 2024-04-26 04:56:06.006636539 +0000 UTC m=+0.563721000 container remove b6928f8fdfd418064e79d5295c46736c259447daad4febcc0db78e7dfdd52e1c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T04:56:06.336 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 podman[115333]: 2024-04-26 04:56:06.123837803 +0000 UTC m=+0.018973503 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:06.336 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 podman[115333]: 2024-04-26 04:56:06.232704476 +0000 UTC m=+0.127840205 container create 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T04:56:06.602 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 podman[115333]: 2024-04-26 04:56:06.460357056 +0000 UTC m=+0.355492739 container init 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T04:56:06.603 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 podman[115333]: 2024-04-26 04:56:06.464929596 +0000 UTC m=+0.360065271 container start 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T04:56:06.603 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-osd[115354]: -- 172.21.15.5:0/2687025094 <== mon.2 v2:172.21.15.5:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b01b6b680 con 0x555b01b34c00 2024-04-26T04:56:06.741 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 2 2024-04-26T04:56:06.741 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:56:06.741 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1 2024-04-26T04:56:06.741 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:56:06.741 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1 with size 2097152 2024-04-26T04:56:06.742 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2 with size 4194304 2024-04-26T04:56:06.746 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 2 2024-04-26T04:56:06.746 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:56:06.746 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap1-rename 2024-04-26T04:56:06.746 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2 2024-04-26T04:56:06.746 INFO:tasks.workunit.client.0.smithi005.stdout:found snap1-rename with size 2097152 2024-04-26T04:56:06.746 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2 with size 4194304 2024-04-26T04:56:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[91801]: osdmap e612: 8 total, 7 up, 8 in 2024-04-26T04:56:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[95809]: osdmap e612: 8 total, 7 up, 8 in 2024-04-26T04:56:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:06.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 bash[115333]: 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 2024-04-26T04:56:06.980 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:06 smithi005 systemd[1]: Started Ceph osd.2 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:56:07.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:06 smithi111 ceph-mon[86360]: osdmap e612: 8 total, 7 up, 8 in 2024-04-26T04:56:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:07.480 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[115347]: 2024-04-26T04:56:07.007+0000 7ff2e86c47c0 -1 Falling back to public interface 2024-04-26T04:56:07.695 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:56:07.696 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:56:07.696 INFO:tasks.workunit.client.0.smithi005.stdout:snap: snap2-rename 2024-04-26T04:56:07.696 INFO:tasks.workunit.client.0.smithi005.stdout:found snap2-rename with size 4194304 2024-04-26T04:56:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:07 smithi111 ceph-mon[86360]: pgmap v553: 297 pgs: 3 active+clean+remapped, 13 active+undersized+degraded, 73 peering, 1 stale+active+clean, 19 active+undersized, 188 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.5 KiB/s wr, 62 op/s; 21/474 objects degraded (4.430%); 8 B/s, 0 objects/s recovering 2024-04-26T04:56:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:07 smithi111 ceph-mon[86360]: Health check failed: Degraded data redundancy: 21/474 objects degraded (4.430%), 13 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:07 smithi111 ceph-mon[86360]: osdmap e613: 8 total, 7 up, 8 in 2024-04-26T04:56:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:07 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2559977883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:07 smithi111 ceph-mon[86360]: osdmap e614: 8 total, 7 up, 8 in 2024-04-26T04:56:07.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[91801]: pgmap v553: 297 pgs: 3 active+clean+remapped, 13 active+undersized+degraded, 73 peering, 1 stale+active+clean, 19 active+undersized, 188 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.5 KiB/s wr, 62 op/s; 21/474 objects degraded (4.430%); 8 B/s, 0 objects/s recovering 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[91801]: Health check failed: Degraded data redundancy: 21/474 objects degraded (4.430%), 13 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[91801]: osdmap e613: 8 total, 7 up, 8 in 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2559977883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[91801]: osdmap e614: 8 total, 7 up, 8 in 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[95809]: pgmap v553: 297 pgs: 3 active+clean+remapped, 13 active+undersized+degraded, 73 peering, 1 stale+active+clean, 19 active+undersized, 188 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.5 KiB/s wr, 62 op/s; 21/474 objects degraded (4.430%); 8 B/s, 0 objects/s recovering 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[95809]: Health check failed: Degraded data redundancy: 21/474 objects degraded (4.430%), 13 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[95809]: osdmap e613: 8 total, 7 up, 8 in 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2559977883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:07.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:07 smithi005 ceph-mon[95809]: osdmap e614: 8 total, 7 up, 8 in 2024-04-26T04:56:08.688 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:56:08.688 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:56:08.692 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (3967 ms) 2024-04-26T04:56:08.692 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-04-26T04:56:09.950 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[115347]: 2024-04-26T04:56:09.876+0000 7ff2e86c47c0 -1 osd.2 607 log_to_monitors true 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[91801]: pgmap v556: 297 pgs: 5 active+clean+remapped, 35 active+undersized+degraded, 67 active+undersized, 190 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 5.7 KiB/s wr, 69 op/s; 66/480 objects degraded (13.750%); 1/480 objects misplaced (0.208%); 8 B/s, 0 objects/s recovering 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[91801]: osdmap e615: 8 total, 7 up, 8 in 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4075275943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[91801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[95809]: pgmap v556: 297 pgs: 5 active+clean+remapped, 35 active+undersized+degraded, 67 active+undersized, 190 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 5.7 KiB/s wr, 69 op/s; 66/480 objects degraded (13.750%); 1/480 objects misplaced (0.208%); 8 B/s, 0 objects/s recovering 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[95809]: osdmap e615: 8 total, 7 up, 8 in 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4075275943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:09.951 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:09 smithi005 ceph-mon[95809]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-26T04:56:09.966 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:09 smithi111 ceph-mon[86360]: pgmap v556: 297 pgs: 5 active+clean+remapped, 35 active+undersized+degraded, 67 active+undersized, 190 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 5.7 KiB/s wr, 69 op/s; 66/480 objects degraded (13.750%); 1/480 objects misplaced (0.208%); 8 B/s, 0 objects/s recovering 2024-04-26T04:56:09.966 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:09 smithi111 ceph-mon[86360]: osdmap e615: 8 total, 7 up, 8 in 2024-04-26T04:56:09.966 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:09 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4075275943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:09.966 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:09 smithi111 ceph-mon[86360]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-26T04:56:10.700 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:10.700 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:10.700 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:10.700 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:10.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: osdmap e616: 8 total, 7 up, 8 in 2024-04-26T04:56:10.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3234639363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:10.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: osdmap e616: 8 total, 7 up, 8 in 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3234639363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:10.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: osdmap e616: 8 total, 7 up, 8 in 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3234639363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.479 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 04:56:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[115347]: 2024-04-26T04:56:10.986+0000 7ff2d5b14700 -1 osd.2 607 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:56:11.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: pgmap v559: 329 pgs: 32 unknown, 5 active+clean+remapped, 35 active+undersized+degraded, 67 active+undersized, 190 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 5.0 KiB/s wr, 67 op/s; 66/480 objects degraded (13.750%); 1/480 objects misplaced (0.208%); 0 B/s, 0 objects/s recovering 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3234639363' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: osdmap e617: 8 total, 7 up, 8 in 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/662029769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: pgmap v559: 329 pgs: 32 unknown, 5 active+clean+remapped, 35 active+undersized+degraded, 67 active+undersized, 190 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 5.0 KiB/s wr, 67 op/s; 66/480 objects degraded (13.750%); 1/480 objects misplaced (0.208%); 0 B/s, 0 objects/s recovering 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:56:11.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3234639363' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: osdmap e617: 8 total, 7 up, 8 in 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/662029769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:11.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: pgmap v559: 329 pgs: 32 unknown, 5 active+clean+remapped, 35 active+undersized+degraded, 67 active+undersized, 190 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 5.0 KiB/s wr, 67 op/s; 66/480 objects degraded (13.750%); 1/480 objects misplaced (0.208%); 0 B/s, 0 objects/s recovering 2024-04-26T04:56:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:56:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3234639363' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:56:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: osdmap e617: 8 total, 7 up, 8 in 2024-04-26T04:56:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:12.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969]' entity='osd.2' 2024-04-26T04:56:12.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/662029769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:12.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:12.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:12.973 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4281 ms) 2024-04-26T04:56:12.973 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-04-26T04:56:12.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[95809]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:56:12.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[95809]: osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969] boot 2024-04-26T04:56:12.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[95809]: osdmap e618: 8 total, 8 up, 8 in 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[95809]: Health check update: Degraded data redundancy: 66/480 objects degraded (13.750%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[95809]: osdmap e619: 8 total, 8 up, 8 in 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[91801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[91801]: osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969] boot 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[91801]: osdmap e618: 8 total, 8 up, 8 in 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[91801]: Health check update: Degraded data redundancy: 66/480 objects degraded (13.750%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:12.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:12 smithi005 ceph-mon[91801]: osdmap e619: 8 total, 8 up, 8 in 2024-04-26T04:56:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:12 smithi111 ceph-mon[86360]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:56:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:12 smithi111 ceph-mon[86360]: osd.2 [v2:172.21.15.5:6818/1259178969,v1:172.21.15.5:6819/1259178969] boot 2024-04-26T04:56:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:12 smithi111 ceph-mon[86360]: osdmap e618: 8 total, 8 up, 8 in 2024-04-26T04:56:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:12 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:56:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:12 smithi111 ceph-mon[86360]: Health check update: Degraded data redundancy: 66/480 objects degraded (13.750%), 35 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:12 smithi111 ceph-mon[86360]: osdmap e619: 8 total, 8 up, 8 in 2024-04-26T04:56:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:56:13] "GET /metrics HTTP/1.1" 200 51012 "" "Prometheus/2.43.0" 2024-04-26T04:56:13.848 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-mon[91801]: pgmap v562: 329 pgs: 22 unknown, 5 active+clean+remapped, 35 active+undersized+degraded, 71 active+undersized, 196 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s; 66/477 objects degraded (13.836%); 1/477 objects misplaced (0.210%) 2024-04-26T04:56:13.848 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:56:13.848 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-mon[91801]: osdmap e620: 8 total, 8 up, 8 in 2024-04-26T04:56:13.849 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-mon[95809]: pgmap v562: 329 pgs: 22 unknown, 5 active+clean+remapped, 35 active+undersized+degraded, 71 active+undersized, 196 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s; 66/477 objects degraded (13.836%); 1/477 objects misplaced (0.210%) 2024-04-26T04:56:13.849 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:56:13.849 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:13 smithi005 ceph-mon[95809]: osdmap e620: 8 total, 8 up, 8 in 2024-04-26T04:56:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:13 smithi111 ceph-mon[86360]: pgmap v562: 329 pgs: 22 unknown, 5 active+clean+remapped, 35 active+undersized+degraded, 71 active+undersized, 196 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s; 66/477 objects degraded (13.836%); 1/477 objects misplaced (0.210%) 2024-04-26T04:56:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:13 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:56:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:13 smithi111 ceph-mon[86360]: osdmap e620: 8 total, 8 up, 8 in 2024-04-26T04:56:14.717 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:14.839 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3186748685' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:14.839 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:14.839 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3673527399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:14.839 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[91801]: osdmap e621: 8 total, 8 up, 8 in 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3186748685' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3673527399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-mon[95809]: osdmap e621: 8 total, 8 up, 8 in 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:14] ENGINE Bus STOPPING 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:56:14.840 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:14] ENGINE Bus STOPPED 2024-04-26T04:56:14.841 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:14] ENGINE Bus STARTING 2024-04-26T04:56:14.841 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:14] ENGINE Serving on http://:::9283 2024-04-26T04:56:14.841 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:14] ENGINE Bus STARTED 2024-04-26T04:56:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:14 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3186748685' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:14 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:14 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3673527399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:14 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:14 smithi111 ceph-mon[86360]: osdmap e621: 8 total, 8 up, 8 in 2024-04-26T04:56:15.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:56:15.678+0000 7f15dde08700 -1 mgr.server reply reply (11) Resource temporarily unavailable 36 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:56:15.679+0000 7f15dde08700 -1 mgr.server reply reply (16) Device or resource busy 36 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (36 PGs are or would become offline) 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: pgmap v565: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.5 KiB/s wr, 25 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:15.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[91801]: osdmap e622: 8 total, 8 up, 8 in 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: pgmap v565: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.5 KiB/s wr, 25 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:15.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:15.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:15.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:15.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:15.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:15.982 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:15 smithi005 ceph-mon[95809]: osdmap e622: 8 total, 8 up, 8 in 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: pgmap v565: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.5 KiB/s wr, 25 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:16.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:15 smithi111 ceph-mon[86360]: osdmap e622: 8 total, 8 up, 8 in 2024-04-26T04:56:16.790 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:16.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: pgmap v567: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.6 KiB/s wr, 26 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: pgmap v568: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.8 KiB/s wr, 30 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: Upgrade: 36 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4099534296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: pgmap v567: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.6 KiB/s wr, 26 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: pgmap v568: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.8 KiB/s wr, 30 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:16.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: Upgrade: 36 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:56:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4099534296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:56:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:16.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: pgmap v567: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.6 KiB/s wr, 26 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: pgmap v568: 361 pgs: 2 remapped+peering, 207 active+clean, 35 active+undersized, 19 active+undersized+degraded, 59 peering, 3 active+clean+remapped, 36 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.8 KiB/s wr, 30 op/s; 29/474 objects degraded (6.118%) 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: Upgrade: 36 pgs have unknown state; cannot draw any conclusions 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4099534296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:17.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:17 smithi005 ceph-mon[95809]: osdmap e623: 8 total, 8 up, 8 in 2024-04-26T04:56:17.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:17 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-04-26T04:56:17.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:17 smithi005 ceph-mon[95809]: Health check update: Degraded data redundancy: 29/474 objects degraded (6.118%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:17 smithi005 ceph-mon[91801]: osdmap e623: 8 total, 8 up, 8 in 2024-04-26T04:56:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:17 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-04-26T04:56:17.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:17 smithi005 ceph-mon[91801]: Health check update: Degraded data redundancy: 29/474 objects degraded (6.118%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:17 smithi111 ceph-mon[86360]: osdmap e623: 8 total, 8 up, 8 in 2024-04-26T04:56:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:17 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-04-26T04:56:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:17 smithi111 ceph-mon[86360]: Health check update: Degraded data redundancy: 29/474 objects degraded (6.118%), 19 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[95809]: pgmap v571: 361 pgs: 1 remapped+peering, 282 active+clean, 59 peering, 19 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.6 KiB/s wr, 84 op/s; 4 B/s, 0 objects/s recovering 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[95809]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/474 objects degraded (6.118%), 19 pgs degraded) 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[95809]: osdmap e624: 8 total, 8 up, 8 in 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3461340734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[91801]: pgmap v571: 361 pgs: 1 remapped+peering, 282 active+clean, 59 peering, 19 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.6 KiB/s wr, 84 op/s; 4 B/s, 0 objects/s recovering 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[91801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/474 objects degraded (6.118%), 19 pgs degraded) 2024-04-26T04:56:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-04-26T04:56:18.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[91801]: osdmap e624: 8 total, 8 up, 8 in 2024-04-26T04:56:18.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:18 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3461340734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:18 smithi111 ceph-mon[86360]: pgmap v571: 361 pgs: 1 remapped+peering, 282 active+clean, 59 peering, 19 unknown; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.6 KiB/s wr, 84 op/s; 4 B/s, 0 objects/s recovering 2024-04-26T04:56:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:18 smithi111 ceph-mon[86360]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/474 objects degraded (6.118%), 19 pgs degraded) 2024-04-26T04:56:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:18 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-04-26T04:56:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:18 smithi111 ceph-mon[86360]: osdmap e624: 8 total, 8 up, 8 in 2024-04-26T04:56:19.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:18 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3461340734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:20.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:20 smithi005 ceph-mon[95809]: pgmap v573: 361 pgs: 361 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 102 KiB/s rd, 10 KiB/s wr, 149 op/s; 16 B/s, 1 keys/s, 1 objects/s recovering 2024-04-26T04:56:20.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:20 smithi005 ceph-mon[91801]: pgmap v573: 361 pgs: 361 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 102 KiB/s rd, 10 KiB/s wr, 149 op/s; 16 B/s, 1 keys/s, 1 objects/s recovering 2024-04-26T04:56:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:20 smithi111 ceph-mon[86360]: pgmap v573: 361 pgs: 361 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 102 KiB/s rd, 10 KiB/s wr, 149 op/s; 16 B/s, 1 keys/s, 1 objects/s recovering 2024-04-26T04:56:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:21 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/767680498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:21 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/767680498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:22.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:21 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/767680498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:22.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:22 smithi005 ceph-mon[91801]: pgmap v574: 361 pgs: 361 active+clean; 3.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 520 KiB/s wr, 125 op/s; 10 B/s, 1 keys/s, 1 objects/s recovering 2024-04-26T04:56:22.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:22 smithi005 ceph-mon[91801]: osdmap e625: 8 total, 8 up, 8 in 2024-04-26T04:56:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:22 smithi005 ceph-mon[95809]: pgmap v574: 361 pgs: 361 active+clean; 3.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 520 KiB/s wr, 125 op/s; 10 B/s, 1 keys/s, 1 objects/s recovering 2024-04-26T04:56:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:22 smithi005 ceph-mon[95809]: osdmap e625: 8 total, 8 up, 8 in 2024-04-26T04:56:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:22 smithi111 ceph-mon[86360]: pgmap v574: 361 pgs: 361 active+clean; 3.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 520 KiB/s wr, 125 op/s; 10 B/s, 1 keys/s, 1 objects/s recovering 2024-04-26T04:56:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:22 smithi111 ceph-mon[86360]: osdmap e625: 8 total, 8 up, 8 in 2024-04-26T04:56:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:56:23] "GET /metrics HTTP/1.1" 200 55774 "" "Prometheus/2.43.0" 2024-04-26T04:56:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:23 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/628799810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:24.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:23 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/628799810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:24.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:23 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/628799810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:24 smithi111 ceph-mon[86360]: pgmap v576: 361 pgs: 361 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 606 KiB/s wr, 73 op/s; 7 B/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:25.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:24 smithi005 ceph-mon[91801]: pgmap v576: 361 pgs: 361 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 606 KiB/s wr, 73 op/s; 7 B/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:24 smithi005 ceph-mon[95809]: pgmap v576: 361 pgs: 361 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 606 KiB/s wr, 73 op/s; 7 B/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:25 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3657019498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:26.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:25 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3657019498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:26.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:25 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3657019498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:26 smithi111 ceph-mon[86360]: pgmap v577: 361 pgs: 361 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 516 KiB/s wr, 62 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:56:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:26 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:27.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:26 smithi005 ceph-mon[95809]: pgmap v577: 361 pgs: 361 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 516 KiB/s wr, 62 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:56:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:26 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:26 smithi005 ceph-mon[91801]: pgmap v577: 361 pgs: 361 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 516 KiB/s wr, 62 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:56:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:26 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:28 smithi111 ceph-mon[86360]: pgmap v578: 361 pgs: 361 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 523 KiB/s wr, 19 op/s 2024-04-26T04:56:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:28 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1342568987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:28 smithi005 ceph-mon[91801]: pgmap v578: 361 pgs: 361 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 523 KiB/s wr, 19 op/s 2024-04-26T04:56:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:28 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1342568987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:28 smithi005 ceph-mon[95809]: pgmap v578: 361 pgs: 361 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 523 KiB/s wr, 19 op/s 2024-04-26T04:56:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:28 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1342568987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:30 smithi111 ceph-mon[86360]: pgmap v579: 361 pgs: 361 active+clean; 11 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.0 MiB/s wr, 19 op/s 2024-04-26T04:56:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:30 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2659794162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:30 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:30 smithi005 ceph-mon[95809]: pgmap v579: 361 pgs: 361 active+clean; 11 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.0 MiB/s wr, 19 op/s 2024-04-26T04:56:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:30 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2659794162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:30 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:30 smithi005 ceph-mon[91801]: pgmap v579: 361 pgs: 361 active+clean; 11 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.0 MiB/s wr, 19 op/s 2024-04-26T04:56:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:30 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2659794162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:30 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:31.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:31.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[91801]: Upgrade: osd.3 is safe to restart 2024-04-26T04:56:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[95809]: Upgrade: osd.3 is safe to restart 2024-04-26T04:56:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:31 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:56:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:31 smithi111 ceph-mon[86360]: Upgrade: osd.3 is safe to restart 2024-04-26T04:56:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:32.900 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.RemoveFullTry (19927 ms) 2024-04-26T04:56:32.900 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-04-26T04:56:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: pgmap v580: 361 pgs: 361 active+clean; 11 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 717 KiB/s wr, 6 op/s 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4080339758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: pool 'test-librbd-smithi005-77850-23' is full (reached quota's max_bytes: 10 MiB) 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-04-26T04:56:33.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:32 smithi111 ceph-mon[86360]: osdmap e626: 8 total, 8 up, 8 in 2024-04-26T04:56:33.078 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: pgmap v580: 361 pgs: 361 active+clean; 11 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 717 KiB/s wr, 6 op/s 2024-04-26T04:56:33.078 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:33.078 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:56:33.078 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4080339758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: pool 'test-librbd-smithi005-77850-23' is full (reached quota's max_bytes: 10 MiB) 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[91801]: osdmap e626: 8 total, 8 up, 8 in 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: pgmap v580: 361 pgs: 361 active+clean; 11 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 717 KiB/s wr, 6 op/s 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4080339758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: pool 'test-librbd-smithi005-77850-23' is full (reached quota's max_bytes: 10 MiB) 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-04-26T04:56:33.079 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:32 smithi005 ceph-mon[95809]: osdmap e626: 8 total, 8 up, 8 in 2024-04-26T04:56:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:56:33] "GET /metrics HTTP/1.1" 200 55845 "" "Prometheus/2.43.0" 2024-04-26T04:56:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:33 smithi005 ceph-mon[95809]: Upgrade: Updating osd.3 2024-04-26T04:56:33.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:33 smithi005 ceph-mon[95809]: Deploying daemon osd.3 on smithi005 2024-04-26T04:56:33.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:33 smithi005 ceph-mon[91801]: Upgrade: Updating osd.3 2024-04-26T04:56:33.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:33 smithi005 ceph-mon[91801]: Deploying daemon osd.3 on smithi005 2024-04-26T04:56:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:33 smithi111 ceph-mon[86360]: Upgrade: Updating osd.3 2024-04-26T04:56:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:33 smithi111 ceph-mon[86360]: Deploying daemon osd.3 on smithi005 2024-04-26T04:56:34.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:34 smithi005 ceph-mon[95809]: pgmap v582: 361 pgs: 361 active+clean; 16 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.1 MiB/s wr, 3 op/s 2024-04-26T04:56:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:34 smithi005 ceph-mon[95809]: osdmap e627: 8 total, 8 up, 8 in 2024-04-26T04:56:34.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:34 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1217618063' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:34.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:34 smithi005 ceph-mon[91801]: pgmap v582: 361 pgs: 361 active+clean; 16 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.1 MiB/s wr, 3 op/s 2024-04-26T04:56:34.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:34 smithi005 ceph-mon[91801]: osdmap e627: 8 total, 8 up, 8 in 2024-04-26T04:56:34.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:34 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1217618063' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:34 smithi111 ceph-mon[86360]: pgmap v582: 361 pgs: 361 active+clean; 16 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.1 MiB/s wr, 3 op/s 2024-04-26T04:56:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:34 smithi111 ceph-mon[86360]: osdmap e627: 8 total, 8 up, 8 in 2024-04-26T04:56:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:34 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1217618063' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:36.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:35 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1217618063' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:36.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:35 smithi005 ceph-mon[91801]: osdmap e628: 8 total, 8 up, 8 in 2024-04-26T04:56:36.034 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:35 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2254439733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:36.034 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:35 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1217618063' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:36.035 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:35 smithi005 ceph-mon[95809]: osdmap e628: 8 total, 8 up, 8 in 2024-04-26T04:56:36.035 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:35 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2254439733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:36.035 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:35 smithi005 systemd[1]: Stopping Ceph osd.3 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:56:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:35 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1217618063' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:35 smithi111 ceph-mon[86360]: osdmap e628: 8 total, 8 up, 8 in 2024-04-26T04:56:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:35 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2254439733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:36.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[58888]: 2024-04-26T04:56:36.038+0000 7f56f5622700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:56:36.480 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[58888]: 2024-04-26T04:56:36.038+0000 7f56f5622700 -1 osd.3 629 *** Got signal Terminated *** 2024-04-26T04:56:36.480 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[58888]: 2024-04-26T04:56:36.038+0000 7f56f5622700 -1 osd.3 629 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:56:36.838 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:37.049 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[95809]: pgmap v585: 393 pgs: 32 unknown, 361 active+clean; 16 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 854 KiB/s wr, 3 op/s 2024-04-26T04:56:37.049 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[95809]: osdmap e629: 8 total, 8 up, 8 in 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/41039136' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[95809]: osd.3 marked itself down and dead 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[91801]: pgmap v585: 393 pgs: 32 unknown, 361 active+clean; 16 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 854 KiB/s wr, 3 op/s 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[91801]: osdmap e629: 8 total, 8 up, 8 in 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/41039136' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:37.050 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-mon[91801]: osd.3 marked itself down and dead 2024-04-26T04:56:37.051 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:36] ENGINE Bus STOPPING 2024-04-26T04:56:37.051 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:56:37.051 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:36] ENGINE Bus STOPPED 2024-04-26T04:56:37.051 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:36] ENGINE Bus STARTING 2024-04-26T04:56:37.051 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:36] ENGINE Serving on http://:::9283 2024-04-26T04:56:37.051 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:36 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:56:36] ENGINE Bus STARTED 2024-04-26T04:56:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:36 smithi111 ceph-mon[86360]: pgmap v585: 393 pgs: 32 unknown, 361 active+clean; 16 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 854 KiB/s wr, 3 op/s 2024-04-26T04:56:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:36 smithi111 ceph-mon[86360]: osdmap e629: 8 total, 8 up, 8 in 2024-04-26T04:56:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:36 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/41039136' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:36 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:36 smithi111 ceph-mon[86360]: osd.3 marked itself down and dead 2024-04-26T04:56:37.632 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:37 smithi005 podman[119831]: 2024-04-26 04:56:37.349372517 +0000 UTC m=+1.412437881 container died fd23fb067fc90a574422ea09c471fb4d54db1df22e6fe17669980898ddee0cb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T04:56:37.982 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:37 smithi005 podman[119831]: 2024-04-26 04:56:37.799936218 +0000 UTC m=+1.863001585 container cleanup fd23fb067fc90a574422ea09c471fb4d54db1df22e6fe17669980898ddee0cb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5) 2024-04-26T04:56:37.982 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:37 smithi005 bash[119831]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: osdmap e630: 8 total, 7 up, 8 in 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: osd.3 marked itself dead as of e630 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3752810145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[95809]: osdmap e631: 8 total, 7 up, 8 in 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:37.983 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: osdmap e630: 8 total, 7 up, 8 in 2024-04-26T04:56:37.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: osd.3 marked itself dead as of e630 2024-04-26T04:56:37.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:56:37.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3752810145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:37.984 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:37 smithi005 ceph-mon[91801]: osdmap e631: 8 total, 7 up, 8 in 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: osdmap e630: 8 total, 7 up, 8 in 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: osd.3 marked itself dead as of e630 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3752810145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:37 smithi111 ceph-mon[86360]: osdmap e631: 8 total, 7 up, 8 in 2024-04-26T04:56:38.426 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:38 smithi005 podman[119897]: 2024-04-26 04:56:38.058272893 +0000 UTC m=+0.707404607 container remove fd23fb067fc90a574422ea09c471fb4d54db1df22e6fe17669980898ddee0cb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T04:56:38.729 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:38 smithi005 podman[119987]: 2024-04-26 04:56:38.330031505 +0000 UTC m=+0.019304332 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:38.730 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:38 smithi005 podman[119987]: 2024-04-26 04:56:38.517662345 +0000 UTC m=+0.206935179 container create 21889ad353194d703033e9a7604aa19b50e4be58a442b41db7176b12144a96e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:56:39.072 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:38 smithi005 ceph-mon[95809]: pgmap v588: 425 pgs: 55 stale+active+clean, 6 creating+peering, 47 unknown, 317 active+clean; 12 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1023 B/s wr, 12 op/s 2024-04-26T04:56:39.072 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:38 smithi005 ceph-mon[95809]: osdmap e632: 8 total, 7 up, 8 in 2024-04-26T04:56:39.072 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:38 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-04-26T04:56:39.073 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:38 smithi005 ceph-mon[91801]: pgmap v588: 425 pgs: 55 stale+active+clean, 6 creating+peering, 47 unknown, 317 active+clean; 12 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1023 B/s wr, 12 op/s 2024-04-26T04:56:39.073 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:38 smithi005 ceph-mon[91801]: osdmap e632: 8 total, 7 up, 8 in 2024-04-26T04:56:39.073 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:38 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-04-26T04:56:39.073 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:38 smithi005 podman[119987]: 2024-04-26 04:56:38.761684709 +0000 UTC m=+0.450957536 container init 21889ad353194d703033e9a7604aa19b50e4be58a442b41db7176b12144a96e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:56:39.073 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:38 smithi005 podman[119987]: 2024-04-26 04:56:38.765168373 +0000 UTC m=+0.454441207 container start 21889ad353194d703033e9a7604aa19b50e4be58a442b41db7176b12144a96e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T04:56:39.073 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:38 smithi005 podman[119987]: 2024-04-26 04:56:38.862126539 +0000 UTC m=+0.551399372 container attach 21889ad353194d703033e9a7604aa19b50e4be58a442b41db7176b12144a96e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T04:56:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:38 smithi111 ceph-mon[86360]: pgmap v588: 425 pgs: 55 stale+active+clean, 6 creating+peering, 47 unknown, 317 active+clean; 12 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1023 B/s wr, 12 op/s 2024-04-26T04:56:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:38 smithi111 ceph-mon[86360]: osdmap e632: 8 total, 7 up, 8 in 2024-04-26T04:56:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:38 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-04-26T04:56:39.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:39 smithi005 podman[119987]: 2024-04-26 04:56:39.071966463 +0000 UTC m=+0.761239290 container died 21889ad353194d703033e9a7604aa19b50e4be58a442b41db7176b12144a96e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:56:40.134 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:39 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1435440702' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:40.135 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:39 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-04-26T04:56:40.135 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:39 smithi005 ceph-mon[95809]: osdmap e633: 8 total, 7 up, 8 in 2024-04-26T04:56:40.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:39 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1435440702' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:40.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:39 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-04-26T04:56:40.135 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:39 smithi005 ceph-mon[91801]: osdmap e633: 8 total, 7 up, 8 in 2024-04-26T04:56:40.135 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:39 smithi005 podman[120012]: 2024-04-26 04:56:39.892316309 +0000 UTC m=+0.814187158 container remove 21889ad353194d703033e9a7604aa19b50e4be58a442b41db7176b12144a96e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-26T04:56:40.135 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:39 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T04:56:40.136 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:39 smithi005 systemd[1]: Stopped Ceph osd.3 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:56:40.136 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:39 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.3.service: Consumed 27.222s CPU time. 2024-04-26T04:56:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:39 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1435440702' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:39 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi005-77850-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-04-26T04:56:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:39 smithi111 ceph-mon[86360]: osdmap e633: 8 total, 7 up, 8 in 2024-04-26T04:56:40.475 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:40 smithi005 systemd[1]: Starting Ceph osd.3 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T04:56:40.475 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:40 smithi005 podman[120144]: 2024-04-26 04:56:40.256693875 +0000 UTC m=+0.018633590 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:40.475 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:40 smithi005 podman[120144]: 2024-04-26 04:56:40.368287667 +0000 UTC m=+0.130227406 container create 03c8b034ba1ce2935bc5e91001b0e50ee8cd7067f89063ceea0a491fac99ef43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb) 2024-04-26T04:56:40.730 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:40 smithi005 podman[120144]: 2024-04-26 04:56:40.604580601 +0000 UTC m=+0.366520311 container init 03c8b034ba1ce2935bc5e91001b0e50ee8cd7067f89063ceea0a491fac99ef43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T04:56:40.730 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:40 smithi005 podman[120144]: 2024-04-26 04:56:40.608092966 +0000 UTC m=+0.370032667 container start 03c8b034ba1ce2935bc5e91001b0e50ee8cd7067f89063ceea0a491fac99ef43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:56:40.730 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:40 smithi005 podman[120144]: 2024-04-26 04:56:40.6626329 +0000 UTC m=+0.424572600 container attach 03c8b034ba1ce2935bc5e91001b0e50ee8cd7067f89063ceea0a491fac99ef43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-26T04:56:41.095 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:40 smithi005 ceph-mon[91801]: pgmap v591: 425 pgs: 93 peering, 25 stale+active+clean, 12 creating+peering, 5 unknown, 290 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 2.0 KiB/s wr, 12 op/s 2024-04-26T04:56:41.095 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:40 smithi005 ceph-mon[91801]: Health check failed: Reduced data availability: 12 pgs inactive, 18 pgs peering (PG_AVAILABILITY) 2024-04-26T04:56:41.096 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:40 smithi005 ceph-mon[95809]: pgmap v591: 425 pgs: 93 peering, 25 stale+active+clean, 12 creating+peering, 5 unknown, 290 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 2.0 KiB/s wr, 12 op/s 2024-04-26T04:56:41.096 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:40 smithi005 ceph-mon[95809]: Health check failed: Reduced data availability: 12 pgs inactive, 18 pgs peering (PG_AVAILABILITY) 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 bash[120144]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T04:56:41.096 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T04:56:41.097 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T04:56:41.097 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T04:56:41.097 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:56:41.097 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate[120159]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T04:56:41.097 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 podman[120144]: 2024-04-26 04:56:41.095061662 +0000 UTC m=+0.857001359 container died 03c8b034ba1ce2935bc5e91001b0e50ee8cd7067f89063ceea0a491fac99ef43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:56:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:40 smithi111 ceph-mon[86360]: pgmap v591: 425 pgs: 93 peering, 25 stale+active+clean, 12 creating+peering, 5 unknown, 290 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 2.0 KiB/s wr, 12 op/s 2024-04-26T04:56:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:40 smithi111 ceph-mon[86360]: Health check failed: Reduced data availability: 12 pgs inactive, 18 pgs peering (PG_AVAILABILITY) 2024-04-26T04:56:41.968 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-mon[91801]: Health check failed: Degraded data redundancy: 36/501 objects degraded (7.186%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:41.968 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4177339560' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:41.969 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-mon[95809]: Health check failed: Degraded data redundancy: 36/501 objects degraded (7.186%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:41.969 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:41 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4177339560' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:41.969 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 podman[120276]: 2024-04-26 04:56:41.700337471 +0000 UTC m=+0.599520757 container remove 03c8b034ba1ce2935bc5e91001b0e50ee8cd7067f89063ceea0a491fac99ef43 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T04:56:41.969 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 podman[120362]: 2024-04-26 04:56:41.844573701 +0000 UTC m=+0.026516749 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:41.969 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:41 smithi005 podman[120362]: 2024-04-26 04:56:41.968256141 +0000 UTC m=+0.150199214 container create d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:56:42.219 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:42 smithi005 podman[120362]: 2024-04-26 04:56:42.195388389 +0000 UTC m=+0.377331421 container init d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-26T04:56:42.220 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:42 smithi005 podman[120362]: 2024-04-26 04:56:42.198384296 +0000 UTC m=+0.380327328 container start d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:56:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:41 smithi111 ceph-mon[86360]: Health check failed: Degraded data redundancy: 36/501 objects degraded (7.186%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:41 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4177339560' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:42.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-osd[120384]: -- 172.21.15.5:0/3271510406 <== mon.0 v2:172.21.15.5:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559a6c645680 con 0x559a6c60ec00 2024-04-26T04:56:42.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:42 smithi005 bash[120362]: d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c 2024-04-26T04:56:42.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:42 smithi005 systemd[1]: Started Ceph osd.3 for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T04:56:43.196 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T04:56:42.736+0000 7f9cfde227c0 -1 Falling back to public interface 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: pgmap v593: 425 pgs: 25 active+undersized+degraded, 57 active+undersized, 93 peering, 6 creating+peering, 244 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s; 36/501 objects degraded (7.186%) 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: pool 'test-librbd-smithi005-77850-23' no longer out of quota; removing NO_QUOTA flag 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-04-26T04:56:43.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[91801]: osdmap e634: 8 total, 7 up, 8 in 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: pgmap v593: 425 pgs: 25 active+undersized+degraded, 57 active+undersized, 93 peering, 6 creating+peering, 244 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s; 36/501 objects degraded (7.186%) 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: pool 'test-librbd-smithi005-77850-23' no longer out of quota; removing NO_QUOTA flag 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-04-26T04:56:43.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:42 smithi005 ceph-mon[95809]: osdmap e634: 8 total, 7 up, 8 in 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: pgmap v593: 425 pgs: 25 active+undersized+degraded, 57 active+undersized, 93 peering, 6 creating+peering, 244 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s; 36/501 objects degraded (7.186%) 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: pool 'test-librbd-smithi005-77850-23' no longer out of quota; removing NO_QUOTA flag 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-04-26T04:56:43.199 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:42 smithi111 ceph-mon[86360]: osdmap e634: 8 total, 7 up, 8 in 2024-04-26T04:56:43.468 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:56:43] "GET /metrics HTTP/1.1" 200 55845 "" "Prometheus/2.43.0" 2024-04-26T04:56:44.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:43 smithi005 ceph-mon[91801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 12 pgs inactive, 19 pgs peering) 2024-04-26T04:56:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:43 smithi005 ceph-mon[95809]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 12 pgs inactive, 19 pgs peering) 2024-04-26T04:56:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:43 smithi111 ceph-mon[86360]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 12 pgs inactive, 19 pgs peering) 2024-04-26T04:56:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:44 smithi005 ceph-mon[91801]: pgmap v595: 425 pgs: 48 active+undersized+degraded, 133 active+undersized, 244 active+clean; 3.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 535 KiB/s wr, 5 op/s; 67/513 objects degraded (13.060%) 2024-04-26T04:56:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:44 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/778242348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:44 smithi005 ceph-mon[95809]: pgmap v595: 425 pgs: 48 active+undersized+degraded, 133 active+undersized, 244 active+clean; 3.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 535 KiB/s wr, 5 op/s; 67/513 objects degraded (13.060%) 2024-04-26T04:56:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:44 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/778242348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:44 smithi111 ceph-mon[86360]: pgmap v595: 425 pgs: 48 active+undersized+degraded, 133 active+undersized, 244 active+clean; 3.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 535 KiB/s wr, 5 op/s; 67/513 objects degraded (13.060%) 2024-04-26T04:56:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:44 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/778242348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:45.951 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:45 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T04:56:45.643+0000 7f9cfde227c0 -1 osd.3 630 log_to_monitors true 2024-04-26T04:56:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:45 smithi005 ceph-mon[91801]: from='osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:56:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:45 smithi005 ceph-mon[91801]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:56:46.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:45 smithi005 ceph-mon[95809]: from='osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:56:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:45 smithi005 ceph-mon[95809]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:56:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:45 smithi111 ceph-mon[86360]: from='osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:56:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:45 smithi111 ceph-mon[86360]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:56:46.729 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T04:56:46.260+0000 7f9ceb272700 -1 osd.3 630 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:56:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: pgmap v596: 425 pgs: 48 active+undersized+degraded, 133 active+undersized, 244 active+clean; 3.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 457 KiB/s wr, 5 op/s; 67/513 objects degraded (13.060%) 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: osdmap e635: 8 total, 7 up, 8 in 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='osd.3 ' entity='osd.3' 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1193779086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: pgmap v596: 425 pgs: 48 active+undersized+degraded, 133 active+undersized, 244 active+clean; 3.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 457 KiB/s wr, 5 op/s; 67/513 objects degraded (13.060%) 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: osdmap e635: 8 total, 7 up, 8 in 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='osd.3 ' entity='osd.3' 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1193779086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: pgmap v596: 425 pgs: 48 active+undersized+degraded, 133 active+undersized, 244 active+clean; 3.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 457 KiB/s wr, 5 op/s; 67/513 objects degraded (13.060%) 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: osdmap e635: 8 total, 7 up, 8 in 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi005", "root=default"]}]: dispatch 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='osd.3 ' entity='osd.3' 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1193779086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:47.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:47.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157] boot 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: osdmap e636: 8 total, 8 up, 8 in 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[91801]: Health check update: Degraded data redundancy: 67/513 objects degraded (13.060%), 48 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:48.209 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:56:48.210 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157] boot 2024-04-26T04:56:48.210 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: osdmap e636: 8 total, 8 up, 8 in 2024-04-26T04:56:48.210 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:56:48.210 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:48.210 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:48.210 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:47 smithi005 ceph-mon[95809]: Health check update: Degraded data redundancy: 67/513 objects degraded (13.060%), 48 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: osd.3 [v2:172.21.15.5:6826/2840627157,v1:172.21.15.5:6827/2840627157] boot 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: osdmap e636: 8 total, 8 up, 8 in 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:47 smithi111 ceph-mon[86360]: Health check update: Degraded data redundancy: 67/513 objects degraded (13.060%), 48 pgs degraded (PG_DEGRADED) 2024-04-26T04:56:48.901 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T04:56:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:48 smithi005 ceph-mon[91801]: pgmap v599: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 5.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 854 KiB/s wr, 4 op/s; 68/516 objects degraded (13.178%) 2024-04-26T04:56:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:48 smithi005 ceph-mon[91801]: osdmap e637: 8 total, 8 up, 8 in 2024-04-26T04:56:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:48 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2223493313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:48 smithi005 ceph-mon[95809]: pgmap v599: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 5.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 854 KiB/s wr, 4 op/s; 68/516 objects degraded (13.178%) 2024-04-26T04:56:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:48 smithi005 ceph-mon[95809]: osdmap e637: 8 total, 8 up, 8 in 2024-04-26T04:56:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:48 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2223493313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:48 smithi111 ceph-mon[86360]: pgmap v599: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 5.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 854 KiB/s wr, 4 op/s; 68/516 objects degraded (13.178%) 2024-04-26T04:56:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:48 smithi111 ceph-mon[86360]: osdmap e637: 8 total, 8 up, 8 in 2024-04-26T04:56:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:48 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2223493313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:50.020 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:56:50.290 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:50 smithi005 ceph-mon[91801]: pgmap v601: 425 pgs: 66 peering, 34 active+undersized+degraded, 81 active+undersized, 244 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 512 KiB/s wr, 1 op/s; 46/516 objects degraded (8.915%) 2024-04-26T04:56:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:50 smithi005 ceph-mon[95809]: pgmap v601: 425 pgs: 66 peering, 34 active+undersized+degraded, 81 active+undersized, 244 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 512 KiB/s wr, 1 op/s; 46/516 objects degraded (8.915%) 2024-04-26T04:56:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:50 smithi111 ceph-mon[86360]: pgmap v601: 425 pgs: 66 peering, 34 active+undersized+degraded, 81 active+undersized, 244 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 512 KiB/s wr, 1 op/s; 46/516 objects degraded (8.915%) 2024-04-26T04:56:51.979 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:51 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T04:56:51.729+0000 7f15dde08700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (23 PGs are or would become offline) 2024-04-26T04:56:52.759 INFO:teuthology.orchestra.run.smithi005.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:52.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: pgmap v602: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 1.3 MiB/s wr, 2 op/s; 682 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: pgmap v603: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 910 B/s rd, 1.1 MiB/s wr, 2 op/s; 729 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: pgmap v604: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 MiB/s wr, 3 op/s; 948 KiB/s, 2 keys/s, 0 objects/s recovering 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: Upgrade: unsafe to stop osd(s) at this time (23 PGs are or would become offline) 2024-04-26T04:56:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[95809]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/516 objects degraded (8.915%), 34 pgs degraded) 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: pgmap v602: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 1.3 MiB/s wr, 2 op/s; 682 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: pgmap v603: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 910 B/s rd, 1.1 MiB/s wr, 2 op/s; 729 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: pgmap v604: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 MiB/s wr, 3 op/s; 948 KiB/s, 2 keys/s, 0 objects/s recovering 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:52.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:52.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:52.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:56:52.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:56:52.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: Upgrade: unsafe to stop osd(s) at this time (23 PGs are or would become offline) 2024-04-26T04:56:52.982 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:52 smithi005 ceph-mon[91801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/516 objects degraded (8.915%), 34 pgs degraded) 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: pgmap v602: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 1.3 MiB/s wr, 2 op/s; 682 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: pgmap v603: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 910 B/s rd, 1.1 MiB/s wr, 2 op/s; 729 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: pgmap v604: 425 pgs: 66 peering, 359 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 MiB/s wr, 3 op/s; 948 KiB/s, 2 keys/s, 0 objects/s recovering 2024-04-26T04:56:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: Upgrade: unsafe to stop osd(s) at this time (23 PGs are or would become offline) 2024-04-26T04:56:53.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:52 smithi111 ceph-mon[86360]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/516 objects degraded (8.915%), 34 pgs degraded) 2024-04-26T04:56:53.257 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:56:53.449 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (20549 ms) 2024-04-26T04:56:53.449 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestIO 2024-04-26T04:56:53.449 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:56:53] "GET /metrics HTTP/1.1" 200 60520 "" "Prometheus/2.43.0" 2024-04-26T04:56:53.549 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:53.857 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[91801]: pool 'test-librbd-smithi005-77850-25' is full (reached quota's max_bytes: 10 MiB) 2024-04-26T04:56:53.857 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-04-26T04:56:53.857 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[91801]: osdmap e638: 8 total, 8 up, 8 in 2024-04-26T04:56:53.857 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[91801]: from='client.34818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[91801]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[95809]: pool 'test-librbd-smithi005-77850-25' is full (reached quota's max_bytes: 10 MiB) 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[95809]: osdmap e638: 8 total, 8 up, 8 in 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[95809]: from='client.34818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[95809]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:53.858 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:53 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:53 smithi111 ceph-mon[86360]: pool 'test-librbd-smithi005-77850-25' is full (reached quota's max_bytes: 10 MiB) 2024-04-26T04:56:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:53 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-04-26T04:56:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:53 smithi111 ceph-mon[86360]: osdmap e638: 8 total, 8 up, 8 in 2024-04-26T04:56:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:53 smithi111 ceph-mon[86360]: from='client.34818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:53 smithi111 ceph-mon[86360]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:56:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:53 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:56:54.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:54 smithi005 ceph-mon[91801]: pgmap v606: 425 pgs: 425 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 MiB/s wr, 4 op/s; 770 KiB/s, 2 keys/s, 0 objects/s recovering 2024-04-26T04:56:54.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:54 smithi005 ceph-mon[95809]: pgmap v606: 425 pgs: 425 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 MiB/s wr, 4 op/s; 770 KiB/s, 2 keys/s, 0 objects/s recovering 2024-04-26T04:56:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:54 smithi111 ceph-mon[86360]: pgmap v606: 425 pgs: 425 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 MiB/s wr, 4 op/s; 770 KiB/s, 2 keys/s, 0 objects/s recovering 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.475 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.476 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:56:55.477 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.478 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:56:55.479 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:56:55.480 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 512~512 = 512 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 1536~512 = 0 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.481 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:56:55.482 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:56:55.483 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.484 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.485 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.486 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:55.487 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.488 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.489 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:55.490 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:55.490 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:55.490 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:55.490 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:55.490 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:55.490 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:55.588 INFO:teuthology.orchestra.run.smithi005.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:56:55.588 INFO:teuthology.orchestra.run.smithi005.stdout:alertmanager.a smithi005 *:9093,9094 running (8m) 9s ago 12m 24.2M - 0.25.0 c8568f914cd2 3afeca478730 2024-04-26T04:56:55.588 INFO:teuthology.orchestra.run.smithi005.stdout:grafana.a smithi111 *:3000 error 45s ago 12m - - 2024-04-26T04:56:55.588 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi005.zdnlsb smithi005 running (11m) 9s ago 11m 22.2M - 17.2.7-902-gc61472c8 0de407c1b3af 7ddb93a6b5a1 2024-04-26T04:56:55.588 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi111.furhfd smithi111 running (11m) 45s ago 11m 17.8M - 17.2.7-902-gc61472c8 0de407c1b3af 3e206ca72f2e 2024-04-26T04:56:55.588 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.x smithi111 *:8443,9283,8765 running (7m) 45s ago 16m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0572da61dd7 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.y smithi005 *:8443,9283,8765 running (7m) 9s ago 18m 517M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c895550b1a50 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:mon.a smithi005 running (7m) 9s ago 18m 72.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 249028d8f146 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:mon.b smithi111 running (4m) 45s ago 17m 46.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c986c5c74b80 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:mon.c smithi005 running (5m) 9s ago 16m 62.6M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9834a90fbbaa 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.a smithi005 *:9100 running (8m) 9s ago 12m 17.0M - 1.5.0 0da6a335fe13 c74ab514255d 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.b smithi111 *:9100 running (7m) 45s ago 12m 16.2M - 1.5.0 0da6a335fe13 cc0664bb12c9 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.0 smithi005 running (2m) 9s ago 16m 73.2M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 d528575c5fc7 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.1 smithi005 running (114s) 9s ago 15m 71.2M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 d589f860de31 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.2 smithi005 running (49s) 9s ago 15m 62.9M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 2deefb6616c4 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.3 smithi005 running (13s) 9s ago 15m 12.5M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 d52a4be71d7c 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.4 smithi111 running (14m) 45s ago 14m 124M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e2994c2a7aa2 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.5 smithi111 running (14m) 45s ago 14m 110M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5da203330767 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.6 smithi111 running (13m) 45s ago 13m 136M 2480M 17.2.7-902-gc61472c8 0de407c1b3af b7798f2bbff0 2024-04-26T04:56:55.589 INFO:teuthology.orchestra.run.smithi005.stdout:osd.7 smithi111 running (13m) 45s ago 13m 115M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 60f1bafb36a6 2024-04-26T04:56:55.590 INFO:teuthology.orchestra.run.smithi005.stdout:prometheus.a smithi111 *:9095 running (7m) 45s ago 12m 64.9M - 2.43.0 a07b618ecd1d 9e5fc4ef6c62 2024-04-26T04:56:56.242 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:56:56.535 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:56.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:56 smithi005 ceph-mon[91801]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:56.952 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:56 smithi005 ceph-mon[91801]: pgmap v607: 425 pgs: 425 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1018 KiB/s wr, 3 op/s; 678 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:56.952 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:56 smithi005 ceph-mon[95809]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:56.952 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:56 smithi005 ceph-mon[95809]: pgmap v607: 425 pgs: 425 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1018 KiB/s wr, 3 op/s; 678 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:57.017 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:56 smithi111 ceph-mon[86360]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:57.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:56 smithi111 ceph-mon[86360]: pgmap v607: 425 pgs: 425 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1018 KiB/s wr, 3 op/s; 678 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-26T04:56:57.180 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.180 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.180 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.180 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.180 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.180 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.181 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.182 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.183 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.184 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.185 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.186 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.187 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:56:57.188 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestIO (3731 ms) 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-04-26T04:56:57.189 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:56:58.520 INFO:teuthology.orchestra.run.smithi005.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:alertmanager.a smithi005 *:9093,9094 running (8m) 12s ago 12m 24.2M - 0.25.0 c8568f914cd2 3afeca478730 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:grafana.a smithi111 *:3000 error 48s ago 12m - - 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi005.zdnlsb smithi005 running (11m) 12s ago 11m 22.2M - 17.2.7-902-gc61472c8 0de407c1b3af 7ddb93a6b5a1 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mds.foo.smithi111.furhfd smithi111 running (11m) 48s ago 11m 17.8M - 17.2.7-902-gc61472c8 0de407c1b3af 3e206ca72f2e 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.x smithi111 *:8443,9283,8765 running (7m) 48s ago 16m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0572da61dd7 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mgr.y smithi005 *:8443,9283,8765 running (7m) 12s ago 18m 517M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c895550b1a50 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mon.a smithi005 running (7m) 12s ago 18m 72.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 249028d8f146 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mon.b smithi111 running (4m) 48s ago 17m 46.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c986c5c74b80 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:mon.c smithi005 running (5m) 12s ago 16m 62.6M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9834a90fbbaa 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.a smithi005 *:9100 running (8m) 12s ago 12m 17.0M - 1.5.0 0da6a335fe13 c74ab514255d 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:node-exporter.b smithi111 *:9100 running (7m) 48s ago 12m 16.2M - 1.5.0 0da6a335fe13 cc0664bb12c9 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:osd.0 smithi005 running (2m) 12s ago 16m 73.2M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 d528575c5fc7 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:osd.1 smithi005 running (117s) 12s ago 15m 71.2M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 d589f860de31 2024-04-26T04:56:58.521 INFO:teuthology.orchestra.run.smithi005.stdout:osd.2 smithi005 running (52s) 12s ago 15m 62.9M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 2deefb6616c4 2024-04-26T04:56:58.522 INFO:teuthology.orchestra.run.smithi005.stdout:osd.3 smithi005 running (16s) 12s ago 15m 12.5M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 d52a4be71d7c 2024-04-26T04:56:58.522 INFO:teuthology.orchestra.run.smithi005.stdout:osd.4 smithi111 running (14m) 48s ago 14m 124M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e2994c2a7aa2 2024-04-26T04:56:58.522 INFO:teuthology.orchestra.run.smithi005.stdout:osd.5 smithi111 running (14m) 48s ago 14m 110M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 5da203330767 2024-04-26T04:56:58.522 INFO:teuthology.orchestra.run.smithi005.stdout:osd.6 smithi111 running (13m) 48s ago 13m 136M 2480M 17.2.7-902-gc61472c8 0de407c1b3af b7798f2bbff0 2024-04-26T04:56:58.522 INFO:teuthology.orchestra.run.smithi005.stdout:osd.7 smithi111 running (13m) 48s ago 13m 115M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 60f1bafb36a6 2024-04-26T04:56:58.522 INFO:teuthology.orchestra.run.smithi005.stdout:prometheus.a smithi111 *:9095 running (7m) 48s ago 13m 64.9M - 2.43.0 a07b618ecd1d 9e5fc4ef6c62 2024-04-26T04:56:58.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[91801]: pgmap v608: 425 pgs: 425 active+clean; 733 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 524 KiB/s rd, 597 KiB/s wr, 1.00k op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:56:58.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[91801]: pool 'test-librbd-smithi005-77850-25' no longer out of quota; removing NO_QUOTA flag 2024-04-26T04:56:58.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[91801]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-04-26T04:56:58.872 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[91801]: osdmap e639: 8 total, 8 up, 8 in 2024-04-26T04:56:58.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[95809]: pgmap v608: 425 pgs: 425 active+clean; 733 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 524 KiB/s rd, 597 KiB/s wr, 1.00k op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:56:58.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[95809]: pool 'test-librbd-smithi005-77850-25' no longer out of quota; removing NO_QUOTA flag 2024-04-26T04:56:58.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[95809]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-04-26T04:56:58.873 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:58 smithi005 ceph-mon[95809]: osdmap e639: 8 total, 8 up, 8 in 2024-04-26T04:56:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:58 smithi111 ceph-mon[86360]: pgmap v608: 425 pgs: 425 active+clean; 733 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 524 KiB/s rd, 597 KiB/s wr, 1.00k op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:56:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:58 smithi111 ceph-mon[86360]: pool 'test-librbd-smithi005-77850-25' no longer out of quota; removing NO_QUOTA flag 2024-04-26T04:56:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:58 smithi111 ceph-mon[86360]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-04-26T04:56:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:58 smithi111 ceph-mon[86360]: osdmap e639: 8 total, 8 up, 8 in 2024-04-26T04:56:59.180 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-26T04:56:59.449 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:59.893 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:56:59 smithi005 ceph-mon[91801]: from='client.34830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:59.893 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:56:59 smithi005 ceph-mon[95809]: from='client.34830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:56:59 smithi111 ceph-mon[86360]: from='client.34830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:57:00.900 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:00 smithi005 ceph-mon[95809]: pgmap v610: 425 pgs: 425 active+clean; 733 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 396 KiB/s rd, 451 KiB/s wr, 755 op/s; 4 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:00.900 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:00 smithi005 ceph-mon[91801]: pgmap v610: 425 pgs: 425 active+clean; 733 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 396 KiB/s rd, 451 KiB/s wr, 755 op/s; 4 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:00 smithi111 ceph-mon[86360]: pgmap v610: 425 pgs: 425 active+clean; 733 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 396 KiB/s rd, 451 KiB/s wr, 755 op/s; 4 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:01.351 INFO:teuthology.orchestra.run.smithi005.stdout:{ 2024-04-26T04:57:01.351 INFO:teuthology.orchestra.run.smithi005.stdout: "mon": { 2024-04-26T04:57:01.351 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T04:57:01.351 INFO:teuthology.orchestra.run.smithi005.stdout: }, 2024-04-26T04:57:01.351 INFO:teuthology.orchestra.run.smithi005.stdout: "mgr": { 2024-04-26T04:57:01.351 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: }, 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "osd": { 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 4, 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: }, 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "mds": { 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 2 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: }, 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "overall": { 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 6, 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 9 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout: } 2024-04-26T04:57:01.352 INFO:teuthology.orchestra.run.smithi005.stdout:} 2024-04-26T04:57:01.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:01 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2713674495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:01.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:01.919 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:01 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2713674495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:01.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:01 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2713674495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:02.017 DEBUG:teuthology.parallel:result is None 2024-04-26T04:57:02.598 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:02.598 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:02.598 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.599 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:02.600 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:02.601 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:02.602 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:02.603 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 512~512 = 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 1536~512 = 0 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:02.604 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:02.605 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.606 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.607 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.608 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.609 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.610 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.611 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:02.612 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:02.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[91801]: pgmap v611: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 815 KiB/s wr, 976 op/s 2024-04-26T04:57:02.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:02.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[95809]: pgmap v611: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 815 KiB/s wr, 976 op/s 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]: dispatch 2024-04-26T04:57:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:02 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]: dispatch 2024-04-26T04:57:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:02 smithi111 ceph-mon[86360]: pgmap v611: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 815 KiB/s wr, 976 op/s 2024-04-26T04:57:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]: dispatch 2024-04-26T04:57:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]: dispatch 2024-04-26T04:57:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]: dispatch 2024-04-26T04:57:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:02 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]: dispatch 2024-04-26T04:57:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:57:03] "GET /metrics HTTP/1.1" 200 60629 "" "Prometheus/2.43.0" 2024-04-26T04:57:03.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]': finished 2024-04-26T04:57:03.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]': finished 2024-04-26T04:57:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[95809]: osdmap e640: 8 total, 8 up, 8 in 2024-04-26T04:57:03.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[95809]: osdmap e641: 8 total, 8 up, 8 in 2024-04-26T04:57:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]': finished 2024-04-26T04:57:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]': finished 2024-04-26T04:57:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[91801]: osdmap e640: 8 total, 8 up, 8 in 2024-04-26T04:57:03.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:03 smithi005 ceph-mon[91801]: osdmap e641: 8 total, 8 up, 8 in 2024-04-26T04:57:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.1a", "id": [6, 4]}]': finished 2024-04-26T04:57:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.1a", "id": [6, 0]}]': finished 2024-04-26T04:57:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:03 smithi111 ceph-mon[86360]: osdmap e640: 8 total, 8 up, 8 in 2024-04-26T04:57:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:03 smithi111 ceph-mon[86360]: osdmap e641: 8 total, 8 up, 8 in 2024-04-26T04:57:04.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:04 smithi005 ceph-mon[91801]: pgmap v613: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 546 KiB/s rd, 905 KiB/s wr, 1.08k op/s 2024-04-26T04:57:04.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:04 smithi005 ceph-mon[95809]: pgmap v613: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 546 KiB/s rd, 905 KiB/s wr, 1.08k op/s 2024-04-26T04:57:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:04 smithi111 ceph-mon[86360]: pgmap v613: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 546 KiB/s rd, 905 KiB/s wr, 1.08k op/s 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.227 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.228 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.229 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.230 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.231 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.232 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.233 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:06.234 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 4 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:read: 4 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (9047 ms) 2024-04-26T04:57:06.235 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-04-26T04:57:06.236 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:06 smithi005 ceph-mon[91801]: pgmap v615: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 591 KiB/s wr, 336 op/s 2024-04-26T04:57:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:06 smithi005 ceph-mon[95809]: pgmap v615: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 591 KiB/s wr, 336 op/s 2024-04-26T04:57:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:06 smithi111 ceph-mon[86360]: pgmap v615: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 591 KiB/s wr, 336 op/s 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: pgmap v616: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 1.55k op/s 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: pgmap v617: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 813 KiB/s wr, 1.64k op/s 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:08.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:08.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:08.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: pgmap v618: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1003 KiB/s wr, 2.02k op/s 2024-04-26T04:57:08.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: pgmap v619: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 2.55k op/s 2024-04-26T04:57:08.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:07 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: pgmap v616: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 1.55k op/s 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: pgmap v617: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 813 KiB/s wr, 1.64k op/s 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: pgmap v618: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1003 KiB/s wr, 2.02k op/s 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: pgmap v619: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 2.55k op/s 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: pgmap v616: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 1.55k op/s 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: pgmap v617: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 813 KiB/s wr, 1.64k op/s 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:08.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:08.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:08.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: pgmap v618: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1003 KiB/s wr, 2.02k op/s 2024-04-26T04:57:08.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: pgmap v619: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 2.55k op/s 2024-04-26T04:57:08.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:07 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:10.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:09 smithi005 ceph-mon[91801]: pgmap v620: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 810 KiB/s wr, 1.64k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:09 smithi005 ceph-mon[95809]: pgmap v620: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 810 KiB/s wr, 1.64k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:09 smithi111 ceph-mon[86360]: pgmap v620: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 810 KiB/s wr, 1.64k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:12.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:11 smithi005 ceph-mon[91801]: pgmap v621: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 791 KiB/s wr, 1.60k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:11 smithi005 ceph-mon[95809]: pgmap v621: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 791 KiB/s wr, 1.60k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:11 smithi111 ceph-mon[86360]: pgmap v621: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 791 KiB/s wr, 1.60k op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:13.391 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:57:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:57:13] "GET /metrics HTTP/1.1" 200 60629 "" "Prometheus/2.43.0" 2024-04-26T04:57:14.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:13 smithi005 ceph-mon[91801]: pgmap v622: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 666 B/s wr, 9 op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:14.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:13 smithi005 ceph-mon[95809]: pgmap v622: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 666 B/s wr, 9 op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:14.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:13 smithi111 ceph-mon[86360]: pgmap v622: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 666 B/s wr, 9 op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:15.912 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:15.912 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:15.912 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:15.912 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:15.912 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:15.912 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.913 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:15.914 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.915 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.916 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:15.917 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 512~512 = 512 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 1536~512 = 0 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.918 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.919 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.920 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:15.921 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:15.922 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.923 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.924 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:15.925 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:15.926 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:15 smithi005 ceph-mon[91801]: pgmap v623: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 KiB/s rd, 502 B/s wr, 7 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:15 smithi005 ceph-mon[95809]: pgmap v623: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 KiB/s rd, 502 B/s wr, 7 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:16.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:15 smithi111 ceph-mon[86360]: pgmap v623: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 KiB/s rd, 502 B/s wr, 7 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:17.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:17.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:18.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:17 smithi005 ceph-mon[95809]: pgmap v624: 425 pgs: 425 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 410 KiB/s wr, 6 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:17 smithi005 ceph-mon[91801]: pgmap v624: 425 pgs: 425 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 410 KiB/s wr, 6 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:17 smithi111 ceph-mon[86360]: pgmap v624: 425 pgs: 425 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 410 KiB/s wr, 6 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:19.972 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.973 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.974 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.975 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.976 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.977 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.978 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.979 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.980 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.981 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:19.982 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 4 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:read: 4 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (13745 ms) 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-04-26T04:57:19.983 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:20.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:19 smithi005 ceph-mon[91801]: pgmap v625: 425 pgs: 425 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 658 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:19 smithi005 ceph-mon[95809]: pgmap v625: 425 pgs: 425 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 658 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:20.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:19 smithi111 ceph-mon[86360]: pgmap v625: 425 pgs: 425 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 658 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-04-26T04:57:21.993 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:21.994 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:21.995 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:21.996 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:21.997 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:21.998 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:21.999 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:22.000 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 512~512 = 512 2024-04-26T04:57:22.001 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 1536~512 = 0 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.002 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:22.003 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.004 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:22.005 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:22.006 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.007 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.008 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.009 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.010 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.011 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.012 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:22.013 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:22.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:21 smithi005 ceph-mon[91801]: pgmap v626: 425 pgs: 425 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 792 op/s 2024-04-26T04:57:22.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:21 smithi005 ceph-mon[95809]: pgmap v626: 425 pgs: 425 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 792 op/s 2024-04-26T04:57:22.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:21 smithi111 ceph-mon[86360]: pgmap v626: 425 pgs: 425 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.7 MiB/s wr, 792 op/s 2024-04-26T04:57:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:57:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:57:23] "GET /metrics HTTP/1.1" 200 60634 "" "Prometheus/2.43.0" 2024-04-26T04:57:23.764 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.764 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.765 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.766 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.767 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.768 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.769 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.770 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.771 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:23.772 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (3792 ms) 2024-04-26T04:57:23.773 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-04-26T04:57:24.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:24 smithi005 ceph-mon[91801]: pgmap v627: 425 pgs: 425 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.7 MiB/s wr, 863 op/s 2024-04-26T04:57:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:24 smithi005 ceph-mon[95809]: pgmap v627: 425 pgs: 425 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.7 MiB/s wr, 863 op/s 2024-04-26T04:57:24.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:24 smithi111 ceph-mon[86360]: pgmap v627: 425 pgs: 425 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.7 MiB/s wr, 863 op/s 2024-04-26T04:57:25.289 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:25.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[91801]: osdmap e642: 8 total, 8 up, 8 in 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3718092872' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[91801]: pgmap v629: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[95809]: osdmap e642: 8 total, 8 up, 8 in 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3718092872' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:25.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:25 smithi005 ceph-mon[95809]: pgmap v629: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s 2024-04-26T04:57:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:25 smithi111 ceph-mon[86360]: osdmap e642: 8 total, 8 up, 8 in 2024-04-26T04:57:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:25 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3718092872' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:25 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:25 smithi111 ceph-mon[86360]: pgmap v629: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s 2024-04-26T04:57:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:26 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:26 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:26.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:26 smithi005 ceph-mon[91801]: osdmap e643: 8 total, 8 up, 8 in 2024-04-26T04:57:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:26 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:26 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:26.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:26 smithi005 ceph-mon[95809]: osdmap e643: 8 total, 8 up, 8 in 2024-04-26T04:57:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:26 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:26 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:26.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:26 smithi111 ceph-mon[86360]: osdmap e643: 8 total, 8 up, 8 in 2024-04-26T04:57:27.373 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:27.373 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:27.373 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:27.374 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:27.375 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.376 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 512~512 = 512 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 1536~512 = 0 2024-04-26T04:57:27.377 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestDataPoolIO (3609 ms) 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-04-26T04:57:27.378 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.398 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (26 ms) 2024-04-26T04:57:27.399 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-04-26T04:57:27.399 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.423 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (24 ms) 2024-04-26T04:57:27.424 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-04-26T04:57:27.424 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.450 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (26 ms) 2024-04-26T04:57:27.450 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-04-26T04:57:27.450 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.478 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (28 ms) 2024-04-26T04:57:27.478 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-04-26T04:57:27.478 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.537 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (59 ms) 2024-04-26T04:57:27.537 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-04-26T04:57:27.537 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.595 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (58 ms) 2024-04-26T04:57:27.595 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-04-26T04:57:27.595 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.667 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (72 ms) 2024-04-26T04:57:27.667 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-04-26T04:57:27.667 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:27 smithi005 ceph-mon[91801]: osdmap e644: 8 total, 8 up, 8 in 2024-04-26T04:57:27.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:27 smithi005 ceph-mon[91801]: pgmap v632: 457 pgs: 11 unknown, 446 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 305 KiB/s wr, 1.27k op/s 2024-04-26T04:57:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:27 smithi005 ceph-mon[95809]: osdmap e644: 8 total, 8 up, 8 in 2024-04-26T04:57:27.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:27 smithi005 ceph-mon[95809]: pgmap v632: 457 pgs: 11 unknown, 446 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 305 KiB/s wr, 1.27k op/s 2024-04-26T04:57:27.733 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (66 ms) 2024-04-26T04:57:27.733 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-04-26T04:57:27.733 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:27 smithi111 ceph-mon[86360]: osdmap e644: 8 total, 8 up, 8 in 2024-04-26T04:57:27.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:27 smithi111 ceph-mon[86360]: pgmap v632: 457 pgs: 11 unknown, 446 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 305 KiB/s wr, 1.27k op/s 2024-04-26T04:57:28.026 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (293 ms) 2024-04-26T04:57:28.026 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-04-26T04:57:28.026 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.288 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (262 ms) 2024-04-26T04:57:28.288 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-04-26T04:57:28.288 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.314 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (27 ms) 2024-04-26T04:57:28.314 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-04-26T04:57:28.314 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.341 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (27 ms) 2024-04-26T04:57:28.342 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-04-26T04:57:28.342 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.365 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (23 ms) 2024-04-26T04:57:28.365 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-04-26T04:57:28.365 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.391 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (26 ms) 2024-04-26T04:57:28.391 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-04-26T04:57:28.391 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.412 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.412 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 0~1048576 = 0 2024-04-26T04:57:28.412 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.412 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 0~4194304 = 0 2024-04-26T04:57:28.412 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.412 INFO:tasks.workunit.client.0.smithi005.stdout:aio discard: 3145728~1048576 = 0 2024-04-26T04:57:28.413 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (21 ms) 2024-04-26T04:57:28.413 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestFUA 2024-04-26T04:57:28.413 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.515 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.515 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.515 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.515 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.516 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.517 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.518 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestFUA (103 ms) 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-04-26T04:57:28.519 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.536 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:28.537 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.550 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:28.551 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:start compare and write 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:compare and wrote: 512 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.552 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.553 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:28.554 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.554 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.554 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:28.555 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.555 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:28.555 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.555 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.555 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:28.556 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.556 INFO:tasks.workunit.client.0.smithi005.stdout:started aio compare and write 2024-04-26T04:57:28.556 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.557 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:28.557 INFO:tasks.workunit.client.0.smithi005.stdout:finished aio compare and write 2024-04-26T04:57:28.557 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.557 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.557 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.557 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.558 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.559 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.559 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:28.559 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.559 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.559 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.560 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.560 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:28.560 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:discard: 512~512 2024-04-26T04:57:28.561 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:28.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:28 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:57:28.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:28 smithi005 ceph-mon[91801]: osdmap e645: 8 total, 8 up, 8 in 2024-04-26T04:57:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:28 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:57:28.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:28 smithi005 ceph-mon[95809]: osdmap e645: 8 total, 8 up, 8 in 2024-04-26T04:57:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:28 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:57:28.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:28 smithi111 ceph-mon[86360]: osdmap e645: 8 total, 8 up, 8 in 2024-04-26T04:57:29.501 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:29.501 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.502 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:29.503 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.504 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.505 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:29.506 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.507 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.508 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.509 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.510 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.518 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.519 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.519 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.519 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.519 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.519 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.519 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.541 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.563 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.594 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.594 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.594 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.594 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.594 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.594 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.595 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.624 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.624 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.625 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.668 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.668 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.668 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.668 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.668 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.669 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.669 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.715 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.715 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.716 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.716 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.716 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.716 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.716 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.729 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:29 smithi005 ceph-mon[95809]: pgmap v634: 457 pgs: 457 active+clean; 35 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 728 KiB/s rd, 354 KiB/s wr, 1.51k op/s 2024-04-26T04:57:29.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:29 smithi005 ceph-mon[91801]: pgmap v634: 457 pgs: 457 active+clean; 35 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 728 KiB/s rd, 354 KiB/s wr, 1.51k op/s 2024-04-26T04:57:29.767 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:29 smithi111 ceph-mon[86360]: pgmap v634: 457 pgs: 457 active+clean; 35 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 728 KiB/s rd, 354 KiB/s wr, 1.51k op/s 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.768 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.817 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:29.818 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.818 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.818 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.885 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.885 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.886 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.886 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.886 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.886 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.886 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:29.952 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:29.953 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:29.953 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:29.953 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:29.953 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:29.953 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:29.953 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.026 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:30.026 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:30.026 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:30.026 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:30.026 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.026 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.027 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.095 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.096 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.184 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.264 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.361 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.452 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestIOPP (1937 ms) 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-04-26T04:57:30.453 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.478 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (26 ms) 2024-04-26T04:57:30.478 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-04-26T04:57:30.478 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.504 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (26 ms) 2024-04-26T04:57:30.504 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-04-26T04:57:30.505 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.528 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (24 ms) 2024-04-26T04:57:30.528 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-04-26T04:57:30.528 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.551 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (23 ms) 2024-04-26T04:57:30.551 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-04-26T04:57:30.551 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.574 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (23 ms) 2024-04-26T04:57:30.575 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-04-26T04:57:30.575 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.597 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.601 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (27 ms) 2024-04-26T04:57:30.601 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-04-26T04:57:30.601 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.627 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (26 ms) 2024-04-26T04:57:30.627 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-04-26T04:57:30.627 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.650 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.654 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (27 ms) 2024-04-26T04:57:30.654 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-04-26T04:57:30.654 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.679 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (26 ms) 2024-04-26T04:57:30.680 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-04-26T04:57:30.680 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.700 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.703 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (24 ms) 2024-04-26T04:57:30.703 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-04-26T04:57:30.703 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.731 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (27 ms) 2024-04-26T04:57:30.732 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-04-26T04:57:30.732 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.754 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.757 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (26 ms) 2024-04-26T04:57:30.757 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-04-26T04:57:30.757 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (67 ms) 2024-04-26T04:57:30.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-04-26T04:57:30.824 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.871 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:30.891 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (67 ms) 2024-04-26T04:57:30.891 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-04-26T04:57:30.891 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:30.956 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (65 ms) 2024-04-26T04:57:30.956 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-04-26T04:57:30.956 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:31.005 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.024 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (68 ms) 2024-04-26T04:57:31.024 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-04-26T04:57:31.024 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:31.385 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (361 ms) 2024-04-26T04:57:31.385 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-04-26T04:57:31.385 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:31.660 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.677 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (292 ms) 2024-04-26T04:57:31.677 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-04-26T04:57:31.677 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:31.701 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:31.701 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:31.701 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.702 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:31.703 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.703 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:31.703 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:31.703 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.703 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:31.703 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.704 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:31.704 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:31.704 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.704 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:31.704 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.705 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:31.705 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:31.706 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.706 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:57:31.706 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:31.706 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:31.706 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:read: 512 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:31.707 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:57:31.708 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:57:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:31 smithi005 ceph-mon[91801]: pgmap v635: 457 pgs: 457 active+clean; 51 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.9 MiB/s wr, 1.25k op/s 2024-04-26T04:57:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:31 smithi005 ceph-mon[95809]: pgmap v635: 457 pgs: 457 active+clean; 51 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.9 MiB/s wr, 1.25k op/s 2024-04-26T04:57:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:32.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:31 smithi111 ceph-mon[86360]: pgmap v635: 457 pgs: 457 active+clean; 51 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.9 MiB/s wr, 1.25k op/s 2024-04-26T04:57:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:32.664 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:57:32.664 INFO:tasks.workunit.client.0.smithi005.stdout:finished read 2024-04-26T04:57:32.664 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:32.664 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.664 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 0 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 16384 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.665 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 49152 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 65536 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.666 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 98304 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 114688 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 147456 2024-04-26T04:57:32.667 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 163840 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 196608 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.668 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 212992 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.669 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.670 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.674 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.674 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.674 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.675 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.675 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.675 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.675 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.682 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.682 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.683 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.706 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.706 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.706 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.706 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.707 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.707 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.707 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.731 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.731 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.731 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.731 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.731 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.731 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.732 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.764 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.791 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.791 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.791 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.791 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.792 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.837 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.878 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.878 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.878 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.878 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.878 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.879 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.879 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:32.945 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:33.016 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.017 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.017 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:33.017 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.017 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.017 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.089 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.150 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.151 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.151 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.151 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.151 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.151 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.151 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.221 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.222 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.222 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.222 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.222 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.222 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.222 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.286 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.286 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:33.287 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.288 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.288 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.392 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:57:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:57:33] "GET /metrics HTTP/1.1" 200 63035 "" "Prometheus/2.43.0" 2024-04-26T04:57:33.486 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.487 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.487 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.487 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.487 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.487 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.487 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.592 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.593 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.593 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.593 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.593 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.593 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.593 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:57:33.693 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:57:33.693 INFO:tasks.workunit.client.0.smithi005.stdout:finished writesame 2024-04-26T04:57:33.693 INFO:tasks.workunit.client.0.smithi005.stdout:to verify the data 2024-04-26T04:57:33.693 INFO:tasks.workunit.client.0.smithi005.stdout:verified 2024-04-26T04:57:33.693 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.693 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:started writesame 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:expected fail, finished writesame 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (2016 ms) 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-04-26T04:57:33.694 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:33 smithi005 ceph-mon[91801]: pgmap v636: 457 pgs: 457 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 12 MiB/s wr, 2.79k op/s 2024-04-26T04:57:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:33 smithi005 ceph-mon[95809]: pgmap v636: 457 pgs: 457 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 12 MiB/s wr, 2.79k op/s 2024-04-26T04:57:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:33 smithi111 ceph-mon[86360]: pgmap v636: 457 pgs: 457 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 12 MiB/s wr, 2.79k op/s 2024-04-26T04:57:35.014 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:snap: orig 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:found orig with size 2097152 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:write test data! 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 80 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 2 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:snap: orig 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:snap: written 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:found orig with size 2097152 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:found written with size 2097152 2024-04-26T04:57:35.015 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.016 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.016 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.016 INFO:tasks.workunit.client.0.smithi005.stdout:write to snapshot returned -30 2024-04-26T04:57:35.016 INFO:tasks.workunit.client.0.smithi005.stdout:Read-only file system 2024-04-26T04:57:35.038 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.039 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.039 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 80 2024-04-26T04:57:35.039 INFO:tasks.workunit.client.0.smithi005.stdout:opening testimg@orig 2024-04-26T04:57:35.039 INFO:tasks.workunit.client.0.smithi005.stdout:read: 80 2024-04-26T04:57:35.039 INFO:tasks.workunit.client.0.smithi005.stdout:write to snapshot returned -30 2024-04-26T04:57:35.039 INFO:tasks.workunit.client.0.smithi005.stdout:Read-only file system 2024-04-26T04:57:35.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:34 smithi005 ceph-mon[91801]: osdmap e646: 8 total, 8 up, 8 in 2024-04-26T04:57:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:34 smithi005 ceph-mon[95809]: osdmap e646: 8 total, 8 up, 8 in 2024-04-26T04:57:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:34 smithi111 ceph-mon[86360]: osdmap e646: 8 total, 8 up, 8 in 2024-04-26T04:57:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:35 smithi111 ceph-mon[86360]: pgmap v638: 457 pgs: 457 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 11 MiB/s wr, 1.81k op/s 2024-04-26T04:57:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:35 smithi111 ceph-mon[86360]: osdmap e647: 8 total, 8 up, 8 in 2024-04-26T04:57:36.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:35 smithi005 ceph-mon[91801]: pgmap v638: 457 pgs: 457 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 11 MiB/s wr, 1.81k op/s 2024-04-26T04:57:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:35 smithi005 ceph-mon[91801]: osdmap e647: 8 total, 8 up, 8 in 2024-04-26T04:57:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:35 smithi005 ceph-mon[95809]: pgmap v638: 457 pgs: 457 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 11 MiB/s wr, 1.81k op/s 2024-04-26T04:57:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:35 smithi005 ceph-mon[95809]: osdmap e647: 8 total, 8 up, 8 in 2024-04-26T04:57:37.018 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 2 2024-04-26T04:57:37.018 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:57:37.018 INFO:tasks.workunit.client.0.smithi005.stdout:snap: orig 2024-04-26T04:57:37.018 INFO:tasks.workunit.client.0.smithi005.stdout:snap: written 2024-04-26T04:57:37.018 INFO:tasks.workunit.client.0.smithi005.stdout:found orig with size 2097152 2024-04-26T04:57:37.018 INFO:tasks.workunit.client.0.smithi005.stdout:found written with size 2097152 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 1 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:snap: orig 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:found orig with size 2097152 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3324 ms) 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-04-26T04:57:37.019 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:36 smithi111 ceph-mon[86360]: osdmap e648: 8 total, 8 up, 8 in 2024-04-26T04:57:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:36 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:37.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:36 smithi005 ceph-mon[91801]: osdmap e648: 8 total, 8 up, 8 in 2024-04-26T04:57:37.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:36 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:37.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:36 smithi005 ceph-mon[95809]: osdmap e648: 8 total, 8 up, 8 in 2024-04-26T04:57:37.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:36 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:37 smithi111 ceph-mon[86360]: pgmap v641: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 17 MiB/s wr, 2.36k op/s 2024-04-26T04:57:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:37 smithi111 ceph-mon[86360]: osdmap e649: 8 total, 8 up, 8 in 2024-04-26T04:57:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:37 smithi111 ceph-mon[86360]: osdmap e650: 8 total, 8 up, 8 in 2024-04-26T04:57:38.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:37 smithi005 ceph-mon[91801]: pgmap v641: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 17 MiB/s wr, 2.36k op/s 2024-04-26T04:57:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:37 smithi005 ceph-mon[91801]: osdmap e649: 8 total, 8 up, 8 in 2024-04-26T04:57:38.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:37 smithi005 ceph-mon[91801]: osdmap e650: 8 total, 8 up, 8 in 2024-04-26T04:57:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:37 smithi005 ceph-mon[95809]: pgmap v641: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 17 MiB/s wr, 2.36k op/s 2024-04-26T04:57:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:37 smithi005 ceph-mon[95809]: osdmap e649: 8 total, 8 up, 8 in 2024-04-26T04:57:38.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:37 smithi005 ceph-mon[95809]: osdmap e650: 8 total, 8 up, 8 in 2024-04-26T04:57:38.731 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (1714 ms) 2024-04-26T04:57:38.731 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestClone 2024-04-26T04:57:38.731 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:39.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:39 smithi005 ceph-mon[91801]: osdmap e651: 8 total, 8 up, 8 in 2024-04-26T04:57:39.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:39 smithi005 ceph-mon[91801]: pgmap v645: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.4 MiB/s wr, 1.33k op/s 2024-04-26T04:57:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:39 smithi005 ceph-mon[95809]: osdmap e651: 8 total, 8 up, 8 in 2024-04-26T04:57:39.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:39 smithi005 ceph-mon[95809]: pgmap v645: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.4 MiB/s wr, 1.33k op/s 2024-04-26T04:57:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:39 smithi111 ceph-mon[86360]: osdmap e651: 8 total, 8 up, 8 in 2024-04-26T04:57:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:39 smithi111 ceph-mon[86360]: pgmap v645: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.4 MiB/s wr, 1.33k op/s 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:made parent image "parent" 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:parent has no parent info 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:made snapshot "parent@parent_snap" 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:can't unprotect an unprotected snap 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:can't protect a protected snap 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:made and opened clone "child" 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:read: 8 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:read: 8 2024-04-26T04:57:40.717 INFO:tasks.workunit.client.0.smithi005.stdout:read: 8 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:sizes and overlaps are good between parent and child 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:child image successfully cloned all image-meta pairs 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:sized down clone, changed overlap 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:sized up clone, changed size but not overlap or parent's size 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:can't remove parent while child still exists 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:can't remove parent while still protected 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:removed parent snap after unprotecting 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestClone (1986 ms) 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestClone2 2024-04-26T04:57:40.718 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:40.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:40 smithi005 ceph-mon[91801]: osdmap e652: 8 total, 8 up, 8 in 2024-04-26T04:57:40.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:40 smithi005 ceph-mon[95809]: osdmap e652: 8 total, 8 up, 8 in 2024-04-26T04:57:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:40 smithi111 ceph-mon[86360]: osdmap e652: 8 total, 8 up, 8 in 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:made parent image "parent" 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:parent has no parent info 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:made snapshot "parent@parent_snap" 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:made and opened clone "child" 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:child image successfully cloned all image-meta pairs 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestClone2 (1138 ms) 2024-04-26T04:57:41.855 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ListChildren 2024-04-26T04:57:41.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:41 smithi005 ceph-mon[91801]: osdmap e653: 8 total, 8 up, 8 in 2024-04-26T04:57:41.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:41 smithi005 ceph-mon[91801]: pgmap v648: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 904 KiB/s rd, 293 KiB/s wr, 1.62k op/s 2024-04-26T04:57:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:41 smithi005 ceph-mon[95809]: osdmap e653: 8 total, 8 up, 8 in 2024-04-26T04:57:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:41 smithi005 ceph-mon[95809]: pgmap v648: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 904 KiB/s rd, 293 KiB/s wr, 1.62k op/s 2024-04-26T04:57:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:41 smithi111 ceph-mon[86360]: osdmap e653: 8 total, 8 up, 8 in 2024-04-26T04:57:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:41 smithi111 ceph-mon[86360]: pgmap v648: 457 pgs: 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 904 KiB/s rd, 293 KiB/s wr, 1.62k op/s 2024-04-26T04:57:42.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:42 smithi005 ceph-mon[91801]: osdmap e654: 8 total, 8 up, 8 in 2024-04-26T04:57:42.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:42 smithi005 ceph-mon[91801]: osdmap e655: 8 total, 8 up, 8 in 2024-04-26T04:57:42.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:42 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2948243728' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:42.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:42 smithi005 ceph-mon[95809]: osdmap e654: 8 total, 8 up, 8 in 2024-04-26T04:57:42.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:42 smithi005 ceph-mon[95809]: osdmap e655: 8 total, 8 up, 8 in 2024-04-26T04:57:42.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:42 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2948243728' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:42 smithi111 ceph-mon[86360]: osdmap e654: 8 total, 8 up, 8 in 2024-04-26T04:57:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:42 smithi111 ceph-mon[86360]: osdmap e655: 8 total, 8 up, 8 in 2024-04-26T04:57:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:42 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2948243728' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:57:43] "GET /metrics HTTP/1.1" 200 63035 "" "Prometheus/2.43.0" 2024-04-26T04:57:43.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[95809]: pgmap v651: 489 pgs: 32 unknown, 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 757 KiB/s rd, 54 KiB/s wr, 907 op/s 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[95809]: osdmap e656: 8 total, 8 up, 8 in 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[91801]: pgmap v651: 489 pgs: 32 unknown, 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 757 KiB/s rd, 54 KiB/s wr, 907 op/s 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:43.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:43 smithi005 ceph-mon[91801]: osdmap e656: 8 total, 8 up, 8 in 2024-04-26T04:57:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:43 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:43 smithi111 ceph-mon[86360]: pgmap v651: 489 pgs: 32 unknown, 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 757 KiB/s rd, 54 KiB/s wr, 907 op/s 2024-04-26T04:57:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:43 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:43 smithi111 ceph-mon[86360]: osdmap e656: 8 total, 8 up, 8 in 2024-04-26T04:57:44.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[91801]: osdmap e657: 8 total, 8 up, 8 in 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/417290237' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[95809]: osdmap e657: 8 total, 8 up, 8 in 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/417290237' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:44.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:44 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:44 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:44 smithi111 ceph-mon[86360]: osdmap e657: 8 total, 8 up, 8 in 2024-04-26T04:57:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:44 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/417290237' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:44 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:45.709 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:57:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:45 smithi111 ceph-mon[86360]: pgmap v654: 521 pgs: 64 unknown, 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 31 KiB/s wr, 525 op/s 2024-04-26T04:57:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:45 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:45 smithi111 ceph-mon[86360]: osdmap e658: 8 total, 8 up, 8 in 2024-04-26T04:57:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:45 smithi005 ceph-mon[91801]: pgmap v654: 521 pgs: 64 unknown, 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 31 KiB/s wr, 525 op/s 2024-04-26T04:57:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:45 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:45 smithi005 ceph-mon[91801]: osdmap e658: 8 total, 8 up, 8 in 2024-04-26T04:57:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:45 smithi005 ceph-mon[95809]: pgmap v654: 521 pgs: 64 unknown, 457 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 31 KiB/s wr, 525 op/s 2024-04-26T04:57:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:45 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:45 smithi005 ceph-mon[95809]: osdmap e658: 8 total, 8 up, 8 in 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[91801]: osdmap e659: 8 total, 8 up, 8 in 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[91801]: pgmap v657: 521 pgs: 28 unknown, 493 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[95809]: osdmap e659: 8 total, 8 up, 8 in 2024-04-26T04:57:47.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:47 smithi005 ceph-mon[95809]: pgmap v657: 521 pgs: 28 unknown, 493 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-04-26T04:57:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:47 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:47 smithi111 ceph-mon[86360]: osdmap e659: 8 total, 8 up, 8 in 2024-04-26T04:57:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:47 smithi111 ceph-mon[86360]: pgmap v657: 521 pgs: 28 unknown, 493 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-04-26T04:57:48.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:48 smithi005 ceph-mon[91801]: osdmap e660: 8 total, 8 up, 8 in 2024-04-26T04:57:48.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:48 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:57:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:48 smithi005 ceph-mon[95809]: osdmap e660: 8 total, 8 up, 8 in 2024-04-26T04:57:48.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:48 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:57:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:48 smithi111 ceph-mon[86360]: osdmap e660: 8 total, 8 up, 8 in 2024-04-26T04:57:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:48 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:57:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:49 smithi005 ceph-mon[91801]: osdmap e661: 8 total, 8 up, 8 in 2024-04-26T04:57:49.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:49 smithi005 ceph-mon[91801]: pgmap v660: 521 pgs: 521 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-04-26T04:57:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:49 smithi005 ceph-mon[95809]: osdmap e661: 8 total, 8 up, 8 in 2024-04-26T04:57:49.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:49 smithi005 ceph-mon[95809]: pgmap v660: 521 pgs: 521 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-04-26T04:57:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:49 smithi111 ceph-mon[86360]: osdmap e661: 8 total, 8 up, 8 in 2024-04-26T04:57:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:49 smithi111 ceph-mon[86360]: pgmap v660: 521 pgs: 521 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-04-26T04:57:50.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:50 smithi005 ceph-mon[91801]: osdmap e662: 8 total, 8 up, 8 in 2024-04-26T04:57:50.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:50 smithi005 ceph-mon[95809]: osdmap e662: 8 total, 8 up, 8 in 2024-04-26T04:57:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:50 smithi111 ceph-mon[86360]: osdmap e662: 8 total, 8 up, 8 in 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 1 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:child: image96 2024-04-26T04:57:51.187 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96/86b8c76d9e0c 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.188 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 2 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:child: image96 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96/86b8c76d9e0c 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.189 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.190 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:child: image96 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:child: image98 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96/86b8c76d9e0c 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98/86b84c81e894 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.191 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:child: image96 2024-04-26T04:57:51.192 INFO:tasks.workunit.client.0.smithi005.stdout:child: image98 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96/86b8c76d9e0c 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98/86b84c81e894 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.193 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 4 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 4 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.194 INFO:tasks.workunit.client.0.smithi005.stdout:child: image96 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:child: image98 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:child: image99 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96/86b8c76d9e0c 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98/86b84c81e894 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99/86b8bec935e7 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.195 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.196 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.197 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.197 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.197 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image96 2024-04-26T04:57:51.197 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.803 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 4 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 4 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:child: image96 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:child: image98 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:child: image99 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image96/86b8c76d9e0c 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.804 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98/86b84c81e894 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99/86b8bec935e7 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.805 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image98 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:child: image98 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:child: image99 2024-04-26T04:57:51.806 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image98/86b84c81e894 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99/86b8bec935e7 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.807 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 2 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:child: image99 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-28/image99/86b8bec935e7 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 1 2024-04-26T04:57:51.808 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout:child: image97 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-27/image97/86b82dd9941 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 0 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ListChildren (9948 ms) 2024-04-26T04:57:51.809 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-04-26T04:57:51.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:51 smithi005 ceph-mon[91801]: osdmap e663: 8 total, 8 up, 8 in 2024-04-26T04:57:51.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:51 smithi005 ceph-mon[91801]: pgmap v663: 521 pgs: 521 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 3.5 KiB/s wr, 77 op/s 2024-04-26T04:57:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:51 smithi005 ceph-mon[95809]: osdmap e663: 8 total, 8 up, 8 in 2024-04-26T04:57:51.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:51 smithi005 ceph-mon[95809]: pgmap v663: 521 pgs: 521 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 3.5 KiB/s wr, 77 op/s 2024-04-26T04:57:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:51 smithi111 ceph-mon[86360]: osdmap e663: 8 total, 8 up, 8 in 2024-04-26T04:57:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:51 smithi111 ceph-mon[86360]: pgmap v663: 521 pgs: 521 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 3.5 KiB/s wr, 77 op/s 2024-04-26T04:57:52.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[91801]: osdmap e664: 8 total, 8 up, 8 in 2024-04-26T04:57:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[91801]: osdmap e665: 8 total, 8 up, 8 in 2024-04-26T04:57:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3567786499' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:52.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[95809]: osdmap e664: 8 total, 8 up, 8 in 2024-04-26T04:57:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[95809]: osdmap e665: 8 total, 8 up, 8 in 2024-04-26T04:57:52.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3567786499' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:52.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:52 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:52 smithi111 ceph-mon[86360]: osdmap e664: 8 total, 8 up, 8 in 2024-04-26T04:57:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:52 smithi111 ceph-mon[86360]: osdmap e665: 8 total, 8 up, 8 in 2024-04-26T04:57:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:52 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3567786499' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:52 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:57:53] "GET /metrics HTTP/1.1" 200 67775 "" "Prometheus/2.43.0" 2024-04-26T04:57:53.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-mon[91801]: pgmap v666: 553 pgs: 32 unknown, 521 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 9.2 KiB/s wr, 232 op/s 2024-04-26T04:57:53.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:53.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-mon[91801]: osdmap e666: 8 total, 8 up, 8 in 2024-04-26T04:57:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-mon[95809]: pgmap v666: 553 pgs: 32 unknown, 521 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 9.2 KiB/s wr, 232 op/s 2024-04-26T04:57:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:53.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:53 smithi005 ceph-mon[95809]: osdmap e666: 8 total, 8 up, 8 in 2024-04-26T04:57:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:53 smithi111 ceph-mon[86360]: pgmap v666: 553 pgs: 32 unknown, 521 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 9.2 KiB/s wr, 232 op/s 2024-04-26T04:57:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:53 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:53 smithi111 ceph-mon[86360]: osdmap e666: 8 total, 8 up, 8 in 2024-04-26T04:57:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:54 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:54 smithi111 ceph-mon[86360]: osdmap e667: 8 total, 8 up, 8 in 2024-04-26T04:57:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/48820962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:54 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[91801]: osdmap e667: 8 total, 8 up, 8 in 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/48820962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[95809]: osdmap e667: 8 total, 8 up, 8 in 2024-04-26T04:57:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/48820962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:55.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:54 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:55 smithi111 ceph-mon[86360]: pgmap v669: 585 pgs: 64 unknown, 521 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 5.7 KiB/s wr, 155 op/s 2024-04-26T04:57:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:55 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:56.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:55 smithi111 ceph-mon[86360]: osdmap e668: 8 total, 8 up, 8 in 2024-04-26T04:57:56.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:55 smithi005 ceph-mon[91801]: pgmap v669: 585 pgs: 64 unknown, 521 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 5.7 KiB/s wr, 155 op/s 2024-04-26T04:57:56.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:55 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:56.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:55 smithi005 ceph-mon[91801]: osdmap e668: 8 total, 8 up, 8 in 2024-04-26T04:57:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:55 smithi005 ceph-mon[95809]: pgmap v669: 585 pgs: 64 unknown, 521 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 5.7 KiB/s wr, 155 op/s 2024-04-26T04:57:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:55 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:55 smithi005 ceph-mon[95809]: osdmap e668: 8 total, 8 up, 8 in 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: osdmap e669: 8 total, 8 up, 8 in 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3161483890' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: pgmap v672: 617 pgs: 56 unknown, 561 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 7.0 KiB/s wr, 497 op/s 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[91801]: osdmap e670: 8 total, 8 up, 8 in 2024-04-26T04:57:57.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: osdmap e669: 8 total, 8 up, 8 in 2024-04-26T04:57:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3161483890' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: pgmap v672: 617 pgs: 56 unknown, 561 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 7.0 KiB/s wr, 497 op/s 2024-04-26T04:57:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:57.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:57 smithi005 ceph-mon[95809]: osdmap e670: 8 total, 8 up, 8 in 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: osdmap e669: 8 total, 8 up, 8 in 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3161483890' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: pgmap v672: 617 pgs: 56 unknown, 561 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 7.0 KiB/s wr, 497 op/s 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:57:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:57 smithi111 ceph-mon[86360]: osdmap e670: 8 total, 8 up, 8 in 2024-04-26T04:57:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31", "force_nonempty":""}]: dispatch 2024-04-26T04:57:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31", "force_nonempty":""}]': finished 2024-04-26T04:57:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[95809]: osdmap e671: 8 total, 8 up, 8 in 2024-04-26T04:57:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi005-77850-31", "mode":"writeback"}]: dispatch 2024-04-26T04:57:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31", "force_nonempty":""}]: dispatch 2024-04-26T04:57:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31", "force_nonempty":""}]': finished 2024-04-26T04:57:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[91801]: osdmap e671: 8 total, 8 up, 8 in 2024-04-26T04:57:58.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:58 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi005-77850-31", "mode":"writeback"}]: dispatch 2024-04-26T04:57:59.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:58 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31", "force_nonempty":""}]: dispatch 2024-04-26T04:57:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:58 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31", "force_nonempty":""}]': finished 2024-04-26T04:57:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:58 smithi111 ceph-mon[86360]: osdmap e671: 8 total, 8 up, 8 in 2024-04-26T04:57:59.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:58 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi005-77850-31", "mode":"writeback"}]: dispatch 2024-04-26T04:57:59.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[95809]: pgmap v675: 617 pgs: 20 creating+peering, 12 unknown, 585 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 7.0 KiB/s wr, 497 op/s 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[95809]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi005-77850-31", "mode":"writeback"}]': finished 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[95809]: osdmap e672: 8 total, 8 up, 8 in 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi005-77850-29", "overlaypool":"test-librbd-smithi005-77850-31"}]: dispatch 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[91801]: pgmap v675: 617 pgs: 20 creating+peering, 12 unknown, 585 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 7.0 KiB/s wr, 497 op/s 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[91801]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi005-77850-31", "mode":"writeback"}]': finished 2024-04-26T04:57:59.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[91801]: osdmap e672: 8 total, 8 up, 8 in 2024-04-26T04:57:59.981 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:57:59 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi005-77850-29", "overlaypool":"test-librbd-smithi005-77850-31"}]: dispatch 2024-04-26T04:58:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:59 smithi111 ceph-mon[86360]: pgmap v675: 617 pgs: 20 creating+peering, 12 unknown, 585 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 7.0 KiB/s wr, 497 op/s 2024-04-26T04:58:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:59 smithi111 ceph-mon[86360]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-04-26T04:58:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:59 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi005-77850-31", "mode":"writeback"}]': finished 2024-04-26T04:58:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:59 smithi111 ceph-mon[86360]: osdmap e672: 8 total, 8 up, 8 in 2024-04-26T04:58:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:57:59 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi005-77850-29", "overlaypool":"test-librbd-smithi005-77850-31"}]: dispatch 2024-04-26T04:58:00.710 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:00.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:00 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:58:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:00 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi005-77850-29", "overlaypool":"test-librbd-smithi005-77850-31"}]': finished 2024-04-26T04:58:00.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:00 smithi005 ceph-mon[95809]: osdmap e673: 8 total, 8 up, 8 in 2024-04-26T04:58:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:00 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:58:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:00 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi005-77850-29", "overlaypool":"test-librbd-smithi005-77850-31"}]': finished 2024-04-26T04:58:00.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:00 smithi005 ceph-mon[91801]: osdmap e673: 8 total, 8 up, 8 in 2024-04-26T04:58:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:00 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:58:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:00 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi005-77850-29", "overlaypool":"test-librbd-smithi005-77850-31"}]': finished 2024-04-26T04:58:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:00 smithi111 ceph-mon[86360]: osdmap e673: 8 total, 8 up, 8 in 2024-04-26T04:58:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:01 smithi005 ceph-mon[95809]: pgmap v678: 617 pgs: 20 creating+peering, 5 unknown, 592 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:01 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:01.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:01 smithi005 ceph-mon[91801]: pgmap v678: 617 pgs: 20 creating+peering, 5 unknown, 592 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:01 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:01.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:01 smithi111 ceph-mon[86360]: pgmap v678: 617 pgs: 20 creating+peering, 5 unknown, 592 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:01 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:02.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:02.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:02 smithi005 ceph-mon[95809]: osdmap e674: 8 total, 8 up, 8 in 2024-04-26T04:58:02.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:02 smithi005 ceph-mon[95809]: osdmap e675: 8 total, 8 up, 8 in 2024-04-26T04:58:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:02 smithi005 ceph-mon[91801]: osdmap e674: 8 total, 8 up, 8 in 2024-04-26T04:58:02.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:02 smithi005 ceph-mon[91801]: osdmap e675: 8 total, 8 up, 8 in 2024-04-26T04:58:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:02 smithi111 ceph-mon[86360]: osdmap e674: 8 total, 8 up, 8 in 2024-04-26T04:58:03.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:02 smithi111 ceph-mon[86360]: osdmap e675: 8 total, 8 up, 8 in 2024-04-26T04:58:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:58:03] "GET /metrics HTTP/1.1" 200 75019 "" "Prometheus/2.43.0" 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]: dispatch 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]: dispatch 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]: dispatch 2024-04-26T04:58:04.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]: dispatch 2024-04-26T04:58:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2024-04-26T04:58:04.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:03 smithi111 ceph-mon[86360]: pgmap v681: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:04.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[95809]: pgmap v681: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-04-26T04:58:04.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2024-04-26T04:58:04.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:03 smithi005 ceph-mon[91801]: pgmap v681: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]': finished 2024-04-26T04:58:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]': finished 2024-04-26T04:58:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-04-26T04:58:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]': finished 2024-04-26T04:58:05.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:04 smithi111 ceph-mon[86360]: osdmap e676: 8 total, 8 up, 8 in 2024-04-26T04:58:05.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[91801]: osdmap e676: 8 total, 8 up, 8 in 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]': finished 2024-04-26T04:58:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:04 smithi005 ceph-mon[95809]: osdmap e676: 8 total, 8 up, 8 in 2024-04-26T04:58:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:05 smithi111 ceph-mon[86360]: osdmap e677: 8 total, 8 up, 8 in 2024-04-26T04:58:06.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:05 smithi111 ceph-mon[86360]: pgmap v684: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:05 smithi005 ceph-mon[91801]: osdmap e677: 8 total, 8 up, 8 in 2024-04-26T04:58:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:05 smithi005 ceph-mon[91801]: pgmap v684: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:05 smithi005 ceph-mon[95809]: osdmap e677: 8 total, 8 up, 8 in 2024-04-26T04:58:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:05 smithi005 ceph-mon[95809]: pgmap v684: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:06.274 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 1 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:child: image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101/86b858966692 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.275 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 2 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:child: image101 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101/86b858966692 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.276 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:06.277 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:child: image101 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:child: image103 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101/86b858966692 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103/86b8627ae106 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.278 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:child: image101 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:child: image103 2024-04-26T04:58:06.279 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101/86b858966692 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103/86b8627ae106 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.280 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 4 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 4 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:child: image101 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:child: image103 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:child: image104 2024-04-26T04:58:06.281 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101/86b858966692 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103/86b8627ae106 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104/86b8480ce793 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.282 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:06.283 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:06 smithi111 ceph-mon[86360]: osdmap e678: 8 total, 8 up, 8 in 2024-04-26T04:58:07.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:06 smithi005 ceph-mon[91801]: osdmap e678: 8 total, 8 up, 8 in 2024-04-26T04:58:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:06 smithi005 ceph-mon[95809]: osdmap e678: 8 total, 8 up, 8 in 2024-04-26T04:58:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:07 smithi111 ceph-mon[86360]: osdmap e679: 8 total, 8 up, 8 in 2024-04-26T04:58:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:07 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:07 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi005-77850-29"}]: dispatch 2024-04-26T04:58:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:07 smithi111 ceph-mon[86360]: pgmap v687: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 3.7 KiB/s wr, 98 op/s 2024-04-26T04:58:08.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:07 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:08.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[95809]: osdmap e679: 8 total, 8 up, 8 in 2024-04-26T04:58:08.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:08.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi005-77850-29"}]: dispatch 2024-04-26T04:58:08.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[95809]: pgmap v687: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 3.7 KiB/s wr, 98 op/s 2024-04-26T04:58:08.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:08.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[91801]: osdmap e679: 8 total, 8 up, 8 in 2024-04-26T04:58:08.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:08.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi005-77850-29"}]: dispatch 2024-04-26T04:58:08.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[91801]: pgmap v687: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 3.7 KiB/s wr, 98 op/s 2024-04-26T04:58:08.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:07 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image101 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 4 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 4 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:child: image101 2024-04-26T04:58:08.795 INFO:tasks.workunit.client.0.smithi005.stdout:child: image103 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:child: image104 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image101/86b858966692 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103/86b8627ae106 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104/86b8480ce793 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.796 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image103 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:58:08.797 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:child: image103 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:child: image104 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image103/86b8627ae106 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104/86b8480ce793 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.798 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 2 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 2 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:child: image104 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-30/image104/86b8480ce793 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:trying to find test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:checking test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 1 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 1 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:child: image102 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-29/image102/86b837093ff7 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 0 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ListChildrenTiered (16992 ms) 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.LockingPP 2024-04-26T04:58:08.800 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:08.854 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.LockingPP (59 ms) 2024-04-26T04:58:08.854 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.FlushAio 2024-04-26T04:58:08.854 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.110 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.FlushAio (256 ms) 2024-04-26T04:58:09.110 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-04-26T04:58:09.110 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.136 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:08 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi005-77850-29"}]': finished 2024-04-26T04:58:09.136 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:08 smithi005 ceph-mon[91801]: osdmap e680: 8 total, 8 up, 8 in 2024-04-26T04:58:09.136 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:08 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31"}]: dispatch 2024-04-26T04:58:09.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:08 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi005-77850-29"}]': finished 2024-04-26T04:58:09.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:08 smithi005 ceph-mon[95809]: osdmap e680: 8 total, 8 up, 8 in 2024-04-26T04:58:09.136 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:08 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31"}]: dispatch 2024-04-26T04:58:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:08 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi005-77850-29"}]': finished 2024-04-26T04:58:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:08 smithi111 ceph-mon[86360]: osdmap e680: 8 total, 8 up, 8 in 2024-04-26T04:58:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:08 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31"}]: dispatch 2024-04-26T04:58:09.296 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.FlushAioPP (185 ms) 2024-04-26T04:58:09.296 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-04-26T04:58:09.296 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.318 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (22 ms) 2024-04-26T04:58:09.318 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-04-26T04:58:09.318 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.343 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (25 ms) 2024-04-26T04:58:09.343 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-04-26T04:58:09.343 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.358 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ZeroLengthRead (15 ms) 2024-04-26T04:58:09.358 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-04-26T04:58:09.358 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.396 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.LargeCacheRead (38 ms) 2024-04-26T04:58:09.396 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-04-26T04:58:09.396 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.604 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestPendingAio (208 ms) 2024-04-26T04:58:09.604 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-04-26T04:58:09.604 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:09.903 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:10.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[91801]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-04-26T04:58:10.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31"}]': finished 2024-04-26T04:58:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[91801]: osdmap e681: 8 total, 8 up, 8 in 2024-04-26T04:58:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[91801]: pgmap v690: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 14 KiB/s wr, 460 op/s 2024-04-26T04:58:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[95809]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-04-26T04:58:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31"}]': finished 2024-04-26T04:58:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[95809]: osdmap e681: 8 total, 8 up, 8 in 2024-04-26T04:58:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:09 smithi005 ceph-mon[95809]: pgmap v690: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 14 KiB/s wr, 460 op/s 2024-04-26T04:58:10.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:09 smithi111 ceph-mon[86360]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-04-26T04:58:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:09 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi005-77850-29", "tierpool":"test-librbd-smithi005-77850-31"}]': finished 2024-04-26T04:58:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:09 smithi111 ceph-mon[86360]: osdmap e681: 8 total, 8 up, 8 in 2024-04-26T04:58:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:09 smithi111 ceph-mon[86360]: pgmap v690: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 14 KiB/s wr, 460 op/s 2024-04-26T04:58:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:10 smithi005 ceph-mon[91801]: osdmap e682: 8 total, 8 up, 8 in 2024-04-26T04:58:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:10 smithi005 ceph-mon[95809]: osdmap e682: 8 total, 8 up, 8 in 2024-04-26T04:58:11.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:10 smithi111 ceph-mon[86360]: osdmap e682: 8 total, 8 up, 8 in 2024-04-26T04:58:11.859 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2255 ms) 2024-04-26T04:58:11.860 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-04-26T04:58:11.860 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[95809]: osdmap e683: 8 total, 8 up, 8 in 2024-04-26T04:58:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[95809]: pgmap v693: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 433 KiB/s rd, 80 KiB/s wr, 604 op/s 2024-04-26T04:58:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[91801]: osdmap e683: 8 total, 8 up, 8 in 2024-04-26T04:58:12.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[91801]: pgmap v693: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 433 KiB/s rd, 80 KiB/s wr, 604 op/s 2024-04-26T04:58:12.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:11 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:11 smithi111 ceph-mon[86360]: osdmap e683: 8 total, 8 up, 8 in 2024-04-26T04:58:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:11 smithi111 ceph-mon[86360]: pgmap v693: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 433 KiB/s rd, 80 KiB/s wr, 604 op/s 2024-04-26T04:58:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:11 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:12.835 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:13.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[91801]: osdmap e684: 8 total, 8 up, 8 in 2024-04-26T04:58:13.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:13.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:13.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[91801]: pgmap v695: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 453 KiB/s rd, 96 KiB/s wr, 652 op/s 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[91801]: osdmap e685: 8 total, 8 up, 8 in 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[95809]: osdmap e684: 8 total, 8 up, 8 in 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[95809]: pgmap v695: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 453 KiB/s rd, 96 KiB/s wr, 652 op/s 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:12 smithi005 ceph-mon[95809]: osdmap e685: 8 total, 8 up, 8 in 2024-04-26T04:58:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:12 smithi111 ceph-mon[86360]: osdmap e684: 8 total, 8 up, 8 in 2024-04-26T04:58:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:12 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:12 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:12 smithi111 ceph-mon[86360]: pgmap v695: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 453 KiB/s rd, 96 KiB/s wr, 652 op/s 2024-04-26T04:58:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:12 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:12 smithi111 ceph-mon[86360]: osdmap e685: 8 total, 8 up, 8 in 2024-04-26T04:58:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:58:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:58:13] "GET /metrics HTTP/1.1" 200 75019 "" "Prometheus/2.43.0" 2024-04-26T04:58:14.882 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3023 ms) 2024-04-26T04:58:14.882 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Flatten 2024-04-26T04:58:14.882 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:15.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:14 smithi005 ceph-mon[91801]: osdmap e686: 8 total, 8 up, 8 in 2024-04-26T04:58:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:14 smithi005 ceph-mon[91801]: pgmap v698: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 938 KiB/s wr, 1.02k op/s 2024-04-26T04:58:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:14 smithi005 ceph-mon[95809]: osdmap e686: 8 total, 8 up, 8 in 2024-04-26T04:58:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:14 smithi005 ceph-mon[95809]: pgmap v698: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 938 KiB/s wr, 1.02k op/s 2024-04-26T04:58:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:14 smithi111 ceph-mon[86360]: osdmap e686: 8 total, 8 up, 8 in 2024-04-26T04:58:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:14 smithi111 ceph-mon[86360]: pgmap v698: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 938 KiB/s wr, 1.02k op/s 2024-04-26T04:58:15.939 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Flatten (1056 ms) 2024-04-26T04:58:15.939 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Sparsify 2024-04-26T04:58:15.939 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:15.973 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 32768 2024-04-26T04:58:15.973 INFO:tasks.workunit.client.0.smithi005.stdout:read: 32768 2024-04-26T04:58:15.973 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Sparsify (35 ms) 2024-04-26T04:58:15.973 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-04-26T04:58:15.973 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:16.010 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SparsifyPP (37 ms) 2024-04-26T04:58:16.010 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-04-26T04:58:16.010 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:15 smithi005 ceph-mon[91801]: osdmap e687: 8 total, 8 up, 8 in 2024-04-26T04:58:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:15 smithi005 ceph-mon[95809]: osdmap e687: 8 total, 8 up, 8 in 2024-04-26T04:58:16.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:15 smithi111 ceph-mon[86360]: osdmap e687: 8 total, 8 up, 8 in 2024-04-26T04:58:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:16 smithi111 ceph-mon[86360]: osdmap e688: 8 total, 8 up, 8 in 2024-04-26T04:58:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:16 smithi111 ceph-mon[86360]: pgmap v701: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 760 KiB/s rd, 667 KiB/s wr, 457 op/s 2024-04-26T04:58:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:16 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[91801]: osdmap e688: 8 total, 8 up, 8 in 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[91801]: pgmap v701: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 760 KiB/s rd, 667 KiB/s wr, 457 op/s 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[95809]: osdmap e688: 8 total, 8 up, 8 in 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[95809]: pgmap v701: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 760 KiB/s rd, 667 KiB/s wr, 457 op/s 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:58:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:18.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:17 smithi005 ceph-mon[91801]: osdmap e689: 8 total, 8 up, 8 in 2024-04-26T04:58:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:17 smithi005 ceph-mon[91801]: osdmap e690: 8 total, 8 up, 8 in 2024-04-26T04:58:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:17 smithi005 ceph-mon[95809]: osdmap e689: 8 total, 8 up, 8 in 2024-04-26T04:58:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:17 smithi005 ceph-mon[95809]: osdmap e690: 8 total, 8 up, 8 in 2024-04-26T04:58:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:17 smithi111 ceph-mon[86360]: osdmap e689: 8 total, 8 up, 8 in 2024-04-26T04:58:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:17 smithi111 ceph-mon[86360]: osdmap e690: 8 total, 8 up, 8 in 2024-04-26T04:58:19.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:18 smithi005 ceph-mon[91801]: pgmap v704: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 244 KiB/s rd, 42 KiB/s wr, 304 op/s 2024-04-26T04:58:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:18 smithi005 ceph-mon[91801]: osdmap e691: 8 total, 8 up, 8 in 2024-04-26T04:58:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:18 smithi005 ceph-mon[95809]: pgmap v704: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 244 KiB/s rd, 42 KiB/s wr, 304 op/s 2024-04-26T04:58:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:18 smithi005 ceph-mon[95809]: osdmap e691: 8 total, 8 up, 8 in 2024-04-26T04:58:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:18 smithi111 ceph-mon[86360]: pgmap v704: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 244 KiB/s rd, 42 KiB/s wr, 304 op/s 2024-04-26T04:58:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:18 smithi111 ceph-mon[86360]: osdmap e691: 8 total, 8 up, 8 in 2024-04-26T04:58:20.749 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapshotLimit (4739 ms) 2024-04-26T04:58:20.749 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-04-26T04:58:20.749 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:20.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:20 smithi005 ceph-mon[91801]: osdmap e692: 8 total, 8 up, 8 in 2024-04-26T04:58:20.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:20 smithi005 ceph-mon[91801]: pgmap v707: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 32 KiB/s wr, 190 op/s 2024-04-26T04:58:20.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:20 smithi005 ceph-mon[95809]: osdmap e692: 8 total, 8 up, 8 in 2024-04-26T04:58:20.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:20 smithi005 ceph-mon[95809]: pgmap v707: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 32 KiB/s wr, 190 op/s 2024-04-26T04:58:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:20 smithi111 ceph-mon[86360]: osdmap e692: 8 total, 8 up, 8 in 2024-04-26T04:58:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:20 smithi111 ceph-mon[86360]: pgmap v707: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 32 KiB/s wr, 190 op/s 2024-04-26T04:58:21.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:21 smithi005 ceph-mon[91801]: osdmap e693: 8 total, 8 up, 8 in 2024-04-26T04:58:21.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:21 smithi005 ceph-mon[95809]: osdmap e693: 8 total, 8 up, 8 in 2024-04-26T04:58:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:21 smithi111 ceph-mon[86360]: osdmap e693: 8 total, 8 up, 8 in 2024-04-26T04:58:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:22 smithi111 ceph-mon[86360]: osdmap e694: 8 total, 8 up, 8 in 2024-04-26T04:58:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:22 smithi111 ceph-mon[86360]: pgmap v710: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.5 KiB/s wr, 55 op/s 2024-04-26T04:58:23.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:22 smithi005 ceph-mon[95809]: osdmap e694: 8 total, 8 up, 8 in 2024-04-26T04:58:23.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:22 smithi005 ceph-mon[95809]: pgmap v710: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.5 KiB/s wr, 55 op/s 2024-04-26T04:58:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:22 smithi005 ceph-mon[91801]: osdmap e694: 8 total, 8 up, 8 in 2024-04-26T04:58:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:22 smithi005 ceph-mon[91801]: pgmap v710: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.5 KiB/s wr, 55 op/s 2024-04-26T04:58:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:58:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:58:23] "GET /metrics HTTP/1.1" 200 75099 "" "Prometheus/2.43.0" 2024-04-26T04:58:24.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:23 smithi111 ceph-mon[86360]: osdmap e695: 8 total, 8 up, 8 in 2024-04-26T04:58:24.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:23 smithi005 ceph-mon[91801]: osdmap e695: 8 total, 8 up, 8 in 2024-04-26T04:58:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:23 smithi005 ceph-mon[95809]: osdmap e695: 8 total, 8 up, 8 in 2024-04-26T04:58:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:24 smithi111 ceph-mon[86360]: osdmap e696: 8 total, 8 up, 8 in 2024-04-26T04:58:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:24 smithi111 ceph-mon[86360]: pgmap v713: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 8.2 KiB/s wr, 87 op/s 2024-04-26T04:58:25.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:24 smithi005 ceph-mon[91801]: osdmap e696: 8 total, 8 up, 8 in 2024-04-26T04:58:25.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:24 smithi005 ceph-mon[91801]: pgmap v713: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 8.2 KiB/s wr, 87 op/s 2024-04-26T04:58:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:24 smithi005 ceph-mon[95809]: osdmap e696: 8 total, 8 up, 8 in 2024-04-26T04:58:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:24 smithi005 ceph-mon[95809]: pgmap v713: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 8.2 KiB/s wr, 87 op/s 2024-04-26T04:58:25.770 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5021 ms) 2024-04-26T04:58:25.771 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-04-26T04:58:25.771 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:25.833 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:58:25.833 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (62 ms) 2024-04-26T04:58:25.833 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-04-26T04:58:25.833 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:25.897 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (63 ms) 2024-04-26T04:58:25.897 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-04-26T04:58:25.897 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:25 smithi111 ceph-mon[86360]: osdmap e697: 8 total, 8 up, 8 in 2024-04-26T04:58:26.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:25 smithi005 ceph-mon[91801]: osdmap e697: 8 total, 8 up, 8 in 2024-04-26T04:58:26.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:25 smithi005 ceph-mon[95809]: osdmap e697: 8 total, 8 up, 8 in 2024-04-26T04:58:26.800 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (904 ms) 2024-04-26T04:58:26.800 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-04-26T04:58:26.800 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:26 smithi111 ceph-mon[86360]: osdmap e698: 8 total, 8 up, 8 in 2024-04-26T04:58:27.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:26 smithi111 ceph-mon[86360]: pgmap v716: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 KiB/s wr, 31 op/s 2024-04-26T04:58:27.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:26 smithi005 ceph-mon[91801]: osdmap e698: 8 total, 8 up, 8 in 2024-04-26T04:58:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:26 smithi005 ceph-mon[91801]: pgmap v716: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 KiB/s wr, 31 op/s 2024-04-26T04:58:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:26 smithi005 ceph-mon[95809]: osdmap e698: 8 total, 8 up, 8 in 2024-04-26T04:58:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:26 smithi005 ceph-mon[95809]: pgmap v716: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 KiB/s wr, 31 op/s 2024-04-26T04:58:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:27 smithi111 ceph-mon[86360]: osdmap e699: 8 total, 8 up, 8 in 2024-04-26T04:58:28.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:27 smithi111 ceph-mon[86360]: osdmap e700: 8 total, 8 up, 8 in 2024-04-26T04:58:28.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:27 smithi005 ceph-mon[91801]: osdmap e699: 8 total, 8 up, 8 in 2024-04-26T04:58:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:27 smithi005 ceph-mon[91801]: osdmap e700: 8 total, 8 up, 8 in 2024-04-26T04:58:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:27 smithi005 ceph-mon[95809]: osdmap e699: 8 total, 8 up, 8 in 2024-04-26T04:58:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:27 smithi005 ceph-mon[95809]: osdmap e700: 8 total, 8 up, 8 in 2024-04-26T04:58:28.805 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (2005 ms) 2024-04-26T04:58:28.806 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-04-26T04:58:28.806 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:28.855 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (50 ms) 2024-04-26T04:58:28.855 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-04-26T04:58:28.855 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:58:28.855 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-04-26T04:58:28.855 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-04-26T04:58:28.855 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:29.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:28 smithi111 ceph-mon[86360]: pgmap v719: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.2 KiB/s wr, 128 op/s 2024-04-26T04:58:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:28 smithi005 ceph-mon[91801]: pgmap v719: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.2 KiB/s wr, 128 op/s 2024-04-26T04:58:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:28 smithi005 ceph-mon[95809]: pgmap v719: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.2 KiB/s wr, 128 op/s 2024-04-26T04:58:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:29 smithi111 ceph-mon[86360]: osdmap e701: 8 total, 8 up, 8 in 2024-04-26T04:58:30.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:29 smithi005 ceph-mon[91801]: osdmap e701: 8 total, 8 up, 8 in 2024-04-26T04:58:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:29 smithi005 ceph-mon[95809]: osdmap e701: 8 total, 8 up, 8 in 2024-04-26T04:58:30.794 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:58:30.795 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapRemove2 (1939 ms) 2024-04-26T04:58:30.795 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-04-26T04:58:30.795 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:31.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:30 smithi005 ceph-mon[95809]: osdmap e702: 8 total, 8 up, 8 in 2024-04-26T04:58:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:30 smithi005 ceph-mon[95809]: pgmap v722: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.2 KiB/s wr, 128 op/s 2024-04-26T04:58:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:30 smithi005 ceph-mon[91801]: osdmap e702: 8 total, 8 up, 8 in 2024-04-26T04:58:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:30 smithi005 ceph-mon[91801]: pgmap v722: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.2 KiB/s wr, 128 op/s 2024-04-26T04:58:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:30 smithi111 ceph-mon[86360]: osdmap e702: 8 total, 8 up, 8 in 2024-04-26T04:58:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:30 smithi111 ceph-mon[86360]: pgmap v722: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.2 KiB/s wr, 128 op/s 2024-04-26T04:58:31.825 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1031 ms) 2024-04-26T04:58:31.825 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-04-26T04:58:31.825 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:32.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:31 smithi111 ceph-mon[86360]: osdmap e703: 8 total, 8 up, 8 in 2024-04-26T04:58:32.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:31 smithi005 ceph-mon[95809]: osdmap e703: 8 total, 8 up, 8 in 2024-04-26T04:58:32.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:31 smithi005 ceph-mon[91801]: osdmap e703: 8 total, 8 up, 8 in 2024-04-26T04:58:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:32.828 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1003 ms) 2024-04-26T04:58:32.828 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-04-26T04:58:32.829 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:33.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:32 smithi005 ceph-mon[95809]: osdmap e704: 8 total, 8 up, 8 in 2024-04-26T04:58:33.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:32 smithi005 ceph-mon[95809]: pgmap v725: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 5.5 KiB/s wr, 59 op/s 2024-04-26T04:58:33.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:32 smithi005 ceph-mon[91801]: osdmap e704: 8 total, 8 up, 8 in 2024-04-26T04:58:33.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:32 smithi005 ceph-mon[91801]: pgmap v725: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 5.5 KiB/s wr, 59 op/s 2024-04-26T04:58:33.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:32 smithi111 ceph-mon[86360]: osdmap e704: 8 total, 8 up, 8 in 2024-04-26T04:58:33.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:32 smithi111 ceph-mon[86360]: pgmap v725: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 5.5 KiB/s wr, 59 op/s 2024-04-26T04:58:33.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:58:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:58:33] "GET /metrics HTTP/1.1" 200 75097 "" "Prometheus/2.43.0" 2024-04-26T04:58:33.850 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (1022 ms) 2024-04-26T04:58:33.850 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-04-26T04:58:33.850 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:33 smithi005 ceph-mon[91801]: osdmap e705: 8 total, 8 up, 8 in 2024-04-26T04:58:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:33 smithi005 ceph-mon[95809]: osdmap e705: 8 total, 8 up, 8 in 2024-04-26T04:58:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:33 smithi111 ceph-mon[86360]: osdmap e705: 8 total, 8 up, 8 in 2024-04-26T04:58:34.928 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1078 ms) 2024-04-26T04:58:34.928 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-04-26T04:58:34.928 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:34.967 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (39 ms) 2024-04-26T04:58:34.967 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-04-26T04:58:34.967 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:35.005 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (38 ms) 2024-04-26T04:58:35.006 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-04-26T04:58:35.006 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:35.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:34 smithi005 ceph-mon[95809]: osdmap e706: 8 total, 8 up, 8 in 2024-04-26T04:58:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:34 smithi005 ceph-mon[95809]: pgmap v728: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 15 KiB/s wr, 233 op/s 2024-04-26T04:58:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:34 smithi005 ceph-mon[91801]: osdmap e706: 8 total, 8 up, 8 in 2024-04-26T04:58:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:34 smithi005 ceph-mon[91801]: pgmap v728: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 15 KiB/s wr, 233 op/s 2024-04-26T04:58:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:34 smithi111 ceph-mon[86360]: osdmap e706: 8 total, 8 up, 8 in 2024-04-26T04:58:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:34 smithi111 ceph-mon[86360]: pgmap v728: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 15 KiB/s wr, 233 op/s 2024-04-26T04:58:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:35 smithi005 ceph-mon[91801]: osdmap e707: 8 total, 8 up, 8 in 2024-04-26T04:58:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:35 smithi005 ceph-mon[95809]: osdmap e707: 8 total, 8 up, 8 in 2024-04-26T04:58:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:35 smithi111 ceph-mon[86360]: osdmap e707: 8 total, 8 up, 8 in 2024-04-26T04:58:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:36 smithi005 ceph-mon[91801]: osdmap e708: 8 total, 8 up, 8 in 2024-04-26T04:58:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:36 smithi005 ceph-mon[91801]: pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 231 KiB/s rd, 14 KiB/s wr, 308 op/s 2024-04-26T04:58:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:36 smithi005 ceph-mon[95809]: osdmap e708: 8 total, 8 up, 8 in 2024-04-26T04:58:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:36 smithi005 ceph-mon[95809]: pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 231 KiB/s rd, 14 KiB/s wr, 308 op/s 2024-04-26T04:58:37.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:36 smithi111 ceph-mon[86360]: osdmap e708: 8 total, 8 up, 8 in 2024-04-26T04:58:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:36 smithi111 ceph-mon[86360]: pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 231 KiB/s rd, 14 KiB/s wr, 308 op/s 2024-04-26T04:58:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:37 smithi005 ceph-mon[91801]: osdmap e709: 8 total, 8 up, 8 in 2024-04-26T04:58:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:37 smithi005 ceph-mon[91801]: osdmap e710: 8 total, 8 up, 8 in 2024-04-26T04:58:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:37 smithi005 ceph-mon[95809]: osdmap e709: 8 total, 8 up, 8 in 2024-04-26T04:58:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:37 smithi005 ceph-mon[95809]: osdmap e710: 8 total, 8 up, 8 in 2024-04-26T04:58:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:37 smithi111 ceph-mon[86360]: osdmap e709: 8 total, 8 up, 8 in 2024-04-26T04:58:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:37 smithi111 ceph-mon[86360]: osdmap e710: 8 total, 8 up, 8 in 2024-04-26T04:58:39.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:38 smithi005 ceph-mon[91801]: pgmap v734: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 11 KiB/s wr, 258 op/s 2024-04-26T04:58:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:38 smithi005 ceph-mon[91801]: osdmap e711: 8 total, 8 up, 8 in 2024-04-26T04:58:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:38 smithi005 ceph-mon[95809]: pgmap v734: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 11 KiB/s wr, 258 op/s 2024-04-26T04:58:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:38 smithi005 ceph-mon[95809]: osdmap e711: 8 total, 8 up, 8 in 2024-04-26T04:58:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:38 smithi111 ceph-mon[86360]: pgmap v734: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 11 KiB/s wr, 258 op/s 2024-04-26T04:58:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:38 smithi111 ceph-mon[86360]: osdmap e711: 8 total, 8 up, 8 in 2024-04-26T04:58:40.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:40 smithi005 ceph-mon[91801]: osdmap e712: 8 total, 8 up, 8 in 2024-04-26T04:58:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:40 smithi005 ceph-mon[91801]: pgmap v737: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 6.7 KiB/s wr, 123 op/s 2024-04-26T04:58:40.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:40 smithi005 ceph-mon[95809]: osdmap e712: 8 total, 8 up, 8 in 2024-04-26T04:58:40.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:40 smithi005 ceph-mon[95809]: pgmap v737: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 6.7 KiB/s wr, 123 op/s 2024-04-26T04:58:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:40 smithi111 ceph-mon[86360]: osdmap e712: 8 total, 8 up, 8 in 2024-04-26T04:58:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:40 smithi111 ceph-mon[86360]: pgmap v737: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 6.7 KiB/s wr, 123 op/s 2024-04-26T04:58:41.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:41 smithi005 ceph-mon[91801]: osdmap e713: 8 total, 8 up, 8 in 2024-04-26T04:58:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:41 smithi005 ceph-mon[95809]: osdmap e713: 8 total, 8 up, 8 in 2024-04-26T04:58:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:41 smithi111 ceph-mon[86360]: osdmap e713: 8 total, 8 up, 8 in 2024-04-26T04:58:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:42 smithi111 ceph-mon[86360]: osdmap e714: 8 total, 8 up, 8 in 2024-04-26T04:58:43.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:42 smithi111 ceph-mon[86360]: pgmap v740: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 1023 B/s wr, 8 op/s 2024-04-26T04:58:43.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:42 smithi005 ceph-mon[91801]: osdmap e714: 8 total, 8 up, 8 in 2024-04-26T04:58:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:42 smithi005 ceph-mon[91801]: pgmap v740: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 1023 B/s wr, 8 op/s 2024-04-26T04:58:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:42 smithi005 ceph-mon[95809]: osdmap e714: 8 total, 8 up, 8 in 2024-04-26T04:58:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:42 smithi005 ceph-mon[95809]: pgmap v740: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 1023 B/s wr, 8 op/s 2024-04-26T04:58:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:58:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:58:43] "GET /metrics HTTP/1.1" 200 75097 "" "Prometheus/2.43.0" 2024-04-26T04:58:44.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:43 smithi111 ceph-mon[86360]: osdmap e715: 8 total, 8 up, 8 in 2024-04-26T04:58:44.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:43 smithi005 ceph-mon[91801]: osdmap e715: 8 total, 8 up, 8 in 2024-04-26T04:58:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:43 smithi005 ceph-mon[95809]: osdmap e715: 8 total, 8 up, 8 in 2024-04-26T04:58:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:44 smithi111 ceph-mon[86360]: osdmap e716: 8 total, 8 up, 8 in 2024-04-26T04:58:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:44 smithi111 ceph-mon[86360]: pgmap v743: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 4.2 KiB/s wr, 82 op/s 2024-04-26T04:58:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:44 smithi005 ceph-mon[91801]: osdmap e716: 8 total, 8 up, 8 in 2024-04-26T04:58:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:44 smithi005 ceph-mon[91801]: pgmap v743: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 4.2 KiB/s wr, 82 op/s 2024-04-26T04:58:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:44 smithi005 ceph-mon[95809]: osdmap e716: 8 total, 8 up, 8 in 2024-04-26T04:58:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:44 smithi005 ceph-mon[95809]: pgmap v743: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 4.2 KiB/s wr, 82 op/s 2024-04-26T04:58:45.358 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10352 ms) 2024-04-26T04:58:45.358 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Metadata 2024-04-26T04:58:45.358 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:45.866 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Metadata (509 ms) 2024-04-26T04:58:45.866 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-04-26T04:58:45.866 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:45 smithi111 ceph-mon[86360]: osdmap e717: 8 total, 8 up, 8 in 2024-04-26T04:58:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:45 smithi005 ceph-mon[91801]: osdmap e717: 8 total, 8 up, 8 in 2024-04-26T04:58:46.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:45 smithi005 ceph-mon[95809]: osdmap e717: 8 total, 8 up, 8 in 2024-04-26T04:58:46.881 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.MetadataPP (1015 ms) 2024-04-26T04:58:46.881 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-04-26T04:58:46.881 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:46.987 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.UpdateFeatures (106 ms) 2024-04-26T04:58:46.988 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-04-26T04:58:46.988 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-04-26T04:58:46.988 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-04-26T04:58:46.988 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:46 smithi111 ceph-mon[86360]: osdmap e718: 8 total, 8 up, 8 in 2024-04-26T04:58:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:46 smithi111 ceph-mon[86360]: pgmap v746: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 381 KiB/s rd, 153 KiB/s wr, 615 op/s 2024-04-26T04:58:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:46 smithi005 ceph-mon[91801]: osdmap e718: 8 total, 8 up, 8 in 2024-04-26T04:58:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:46 smithi005 ceph-mon[91801]: pgmap v746: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 381 KiB/s rd, 153 KiB/s wr, 615 op/s 2024-04-26T04:58:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:46 smithi005 ceph-mon[95809]: osdmap e718: 8 total, 8 up, 8 in 2024-04-26T04:58:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:46 smithi005 ceph-mon[95809]: pgmap v746: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 381 KiB/s rd, 153 KiB/s wr, 615 op/s 2024-04-26T04:58:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:47.844 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:58:47.844 INFO:tasks.workunit.client.0.smithi005.stdout:25.00% done 2024-04-26T04:58:47.845 INFO:tasks.workunit.client.0.smithi005.stdout:50.00% done 2024-04-26T04:58:47.845 INFO:tasks.workunit.client.0.smithi005.stdout:75.00% done 2024-04-26T04:58:47.845 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.RebuildObjectMap (857 ms) 2024-04-26T04:58:47.845 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:25.00% done 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:50.00% done 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:75.00% done 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (30 ms) 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-04-26T04:58:47.874 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:47 smithi005 ceph-mon[91801]: osdmap e719: 8 total, 8 up, 8 in 2024-04-26T04:58:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:47 smithi005 ceph-mon[95809]: osdmap e719: 8 total, 8 up, 8 in 2024-04-26T04:58:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:47 smithi111 ceph-mon[86360]: osdmap e719: 8 total, 8 up, 8 in 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:25.00% done 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:50.00% done 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:75.00% done 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CheckObjectMap (982 ms) 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-04-26T04:58:48.856 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:48.884 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.BlockingAIO (28 ms) 2024-04-26T04:58:48.884 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-04-26T04:58:48.884 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:49.008 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (124 ms) 2024-04-26T04:58:49.008 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-04-26T04:58:49.008 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T04:58:49.008 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-04-26T04:58:49.008 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-04-26T04:58:49.008 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:49.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:48 smithi005 ceph-mon[95809]: osdmap e720: 8 total, 8 up, 8 in 2024-04-26T04:58:49.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:48 smithi005 ceph-mon[95809]: pgmap v749: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 562 KiB/s rd, 212 KiB/s wr, 902 op/s 2024-04-26T04:58:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:48 smithi005 ceph-mon[91801]: osdmap e720: 8 total, 8 up, 8 in 2024-04-26T04:58:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:48 smithi005 ceph-mon[91801]: pgmap v749: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 562 KiB/s rd, 212 KiB/s wr, 902 op/s 2024-04-26T04:58:49.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:48 smithi111 ceph-mon[86360]: osdmap e720: 8 total, 8 up, 8 in 2024-04-26T04:58:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:48 smithi111 ceph-mon[86360]: pgmap v749: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 562 KiB/s rd, 212 KiB/s wr, 902 op/s 2024-04-26T04:58:49.879 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (871 ms) 2024-04-26T04:58:49.879 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-04-26T04:58:49.879 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:50.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:49 smithi005 ceph-mon[91801]: osdmap e721: 8 total, 8 up, 8 in 2024-04-26T04:58:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:49 smithi005 ceph-mon[95809]: osdmap e721: 8 total, 8 up, 8 in 2024-04-26T04:58:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:49 smithi111 ceph-mon[86360]: osdmap e721: 8 total, 8 up, 8 in 2024-04-26T04:58:50.836 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (957 ms) 2024-04-26T04:58:50.836 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-04-26T04:58:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:50 smithi005 ceph-mon[91801]: osdmap e722: 8 total, 8 up, 8 in 2024-04-26T04:58:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:50 smithi005 ceph-mon[91801]: pgmap v752: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 237 KiB/s rd, 62 KiB/s wr, 361 op/s 2024-04-26T04:58:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:50 smithi005 ceph-mon[95809]: osdmap e722: 8 total, 8 up, 8 in 2024-04-26T04:58:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:50 smithi005 ceph-mon[95809]: pgmap v752: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 237 KiB/s rd, 62 KiB/s wr, 361 op/s 2024-04-26T04:58:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:50 smithi111 ceph-mon[86360]: osdmap e722: 8 total, 8 up, 8 in 2024-04-26T04:58:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:50 smithi111 ceph-mon[86360]: pgmap v752: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 237 KiB/s rd, 62 KiB/s wr, 361 op/s 2024-04-26T04:58:51.940 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:58:51.940 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:58:51.940 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestImageOptions (1104 ms) 2024-04-26T04:58:51.940 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-04-26T04:58:52.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:51 smithi005 ceph-mon[91801]: osdmap e723: 8 total, 8 up, 8 in 2024-04-26T04:58:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:51 smithi005 ceph-mon[95809]: osdmap e723: 8 total, 8 up, 8 in 2024-04-26T04:58:52.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:51 smithi111 ceph-mon[86360]: osdmap e723: 8 total, 8 up, 8 in 2024-04-26T04:58:52.930 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:100.00% done 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (990 ms) 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-04-26T04:58:52.931 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:52.967 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:wrote: 512 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:58:52.968 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:started write 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:write completion cb called! 2024-04-26T04:58:52.969 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 0 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:finished write 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:58:52.970 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:created completion 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:started read 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:read completion cb called! 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:return value is: 512 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ImagePollIO (38 ms) 2024-04-26T04:58:52.971 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Mirror 2024-04-26T04:58:52.972 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:52.980 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:53.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:52 smithi005 ceph-mon[91801]: osdmap e724: 8 total, 8 up, 8 in 2024-04-26T04:58:53.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:52 smithi005 ceph-mon[91801]: pgmap v755: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 75 KiB/s wr, 368 op/s 2024-04-26T04:58:53.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:52 smithi005 ceph-mon[95809]: osdmap e724: 8 total, 8 up, 8 in 2024-04-26T04:58:53.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:52 smithi005 ceph-mon[95809]: pgmap v755: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 75 KiB/s wr, 368 op/s 2024-04-26T04:58:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:52 smithi111 ceph-mon[86360]: osdmap e724: 8 total, 8 up, 8 in 2024-04-26T04:58:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:52 smithi111 ceph-mon[86360]: pgmap v755: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 75 KiB/s wr, 368 op/s 2024-04-26T04:58:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:58:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:58:53] "GET /metrics HTTP/1.1" 200 75100 "" "Prometheus/2.43.0" 2024-04-26T04:58:53.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Mirror (1007 ms) 2024-04-26T04:58:53.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-04-26T04:58:53.989 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (14 ms) 2024-04-26T04:58:53.989 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-04-26T04:58:54.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:53 smithi005 ceph-mon[91801]: osdmap e725: 8 total, 8 up, 8 in 2024-04-26T04:58:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:53 smithi005 ceph-mon[95809]: osdmap e725: 8 total, 8 up, 8 in 2024-04-26T04:58:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:53 smithi111 ceph-mon[86360]: osdmap e725: 8 total, 8 up, 8 in 2024-04-26T04:58:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: osdmap e726: 8 total, 8 up, 8 in 2024-04-26T04:58:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: pgmap v758: 617 pgs: 617 active+clean; 144 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 471 KiB/s rd, 1.2 MiB/s wr, 612 op/s 2024-04-26T04:58:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-04-26T04:58:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-04-26T04:58:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/604a28ae-c3f3-4ff6-a72a-e67924fc38a1"}]: dispatch 2024-04-26T04:58:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/604a28ae-c3f3-4ff6-a72a-e67924fc38a1"}]': finished 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/41966934-0dcf-4fa4-8e7a-e0cf7f28f0ac"}]: dispatch 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/41966934-0dcf-4fa4-8e7a-e0cf7f28f0ac"}]': finished 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/7cc5c390-8622-425d-a4d6-fd82a618be3f"}]: dispatch 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/7cc5c390-8622-425d-a4d6-fd82a618be3f"}]': finished 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]': finished 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: osdmap e726: 8 total, 8 up, 8 in 2024-04-26T04:58:55.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: pgmap v758: 617 pgs: 617 active+clean; 144 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 471 KiB/s rd, 1.2 MiB/s wr, 612 op/s 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/604a28ae-c3f3-4ff6-a72a-e67924fc38a1"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/604a28ae-c3f3-4ff6-a72a-e67924fc38a1"}]': finished 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/41966934-0dcf-4fa4-8e7a-e0cf7f28f0ac"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/41966934-0dcf-4fa4-8e7a-e0cf7f28f0ac"}]': finished 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/7cc5c390-8622-425d-a4d6-fd82a618be3f"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/7cc5c390-8622-425d-a4d6-fd82a618be3f"}]': finished 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]': finished 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: osdmap e726: 8 total, 8 up, 8 in 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: pgmap v758: 617 pgs: 617 active+clean; 144 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 471 KiB/s rd, 1.2 MiB/s wr, 612 op/s 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/604a28ae-c3f3-4ff6-a72a-e67924fc38a1"}]: dispatch 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/604a28ae-c3f3-4ff6-a72a-e67924fc38a1"}]': finished 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/41966934-0dcf-4fa4-8e7a-e0cf7f28f0ac"}]: dispatch 2024-04-26T04:58:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/41966934-0dcf-4fa4-8e7a-e0cf7f28f0ac"}]': finished 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/7cc5c390-8622-425d-a4d6-fd82a618be3f"}]: dispatch 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/7cc5c390-8622-425d-a4d6-fd82a618be3f"}]': finished 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]: dispatch 2024-04-26T04:58:55.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3948629281' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/dfac18e6-c4a5-4c23-bda1-8be729ecd867"}]': finished 2024-04-26T04:58:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:55 smithi111 ceph-mon[86360]: osdmap e727: 8 total, 8 up, 8 in 2024-04-26T04:58:56.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:55 smithi005 ceph-mon[91801]: osdmap e727: 8 total, 8 up, 8 in 2024-04-26T04:58:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:55 smithi005 ceph-mon[95809]: osdmap e727: 8 total, 8 up, 8 in 2024-04-26T04:58:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:56 smithi111 ceph-mon[86360]: pgmap v760: 617 pgs: 617 active+clean; 146 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 224 KiB/s rd, 1.7 MiB/s wr, 294 op/s 2024-04-26T04:58:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:56 smithi111 ceph-mon[86360]: osdmap e728: 8 total, 8 up, 8 in 2024-04-26T04:58:57.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:56 smithi005 ceph-mon[91801]: pgmap v760: 617 pgs: 617 active+clean; 146 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 224 KiB/s rd, 1.7 MiB/s wr, 294 op/s 2024-04-26T04:58:57.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:56 smithi005 ceph-mon[91801]: osdmap e728: 8 total, 8 up, 8 in 2024-04-26T04:58:57.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:56 smithi005 ceph-mon[95809]: pgmap v760: 617 pgs: 617 active+clean; 146 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 224 KiB/s rd, 1.7 MiB/s wr, 294 op/s 2024-04-26T04:58:57.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:56 smithi005 ceph-mon[95809]: osdmap e728: 8 total, 8 up, 8 in 2024-04-26T04:58:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:57 smithi111 ceph-mon[86360]: osdmap e729: 8 total, 8 up, 8 in 2024-04-26T04:58:58.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:57 smithi005 ceph-mon[91801]: osdmap e729: 8 total, 8 up, 8 in 2024-04-26T04:58:58.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:57 smithi005 ceph-mon[95809]: osdmap e729: 8 total, 8 up, 8 in 2024-04-26T04:58:59.060 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5071 ms) 2024-04-26T04:58:59.060 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-04-26T04:58:59.060 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:58:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:59 smithi111 ceph-mon[86360]: pgmap v763: 617 pgs: 617 active+clean; 147 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 219 KiB/s rd, 892 KiB/s wr, 294 op/s 2024-04-26T04:58:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:58:59 smithi111 ceph-mon[86360]: osdmap e730: 8 total, 8 up, 8 in 2024-04-26T04:58:59.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:59 smithi005 ceph-mon[91801]: pgmap v763: 617 pgs: 617 active+clean; 147 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 219 KiB/s rd, 892 KiB/s wr, 294 op/s 2024-04-26T04:58:59.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:58:59 smithi005 ceph-mon[91801]: osdmap e730: 8 total, 8 up, 8 in 2024-04-26T04:58:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:59 smithi005 ceph-mon[95809]: pgmap v763: 617 pgs: 617 active+clean; 147 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 219 KiB/s rd, 892 KiB/s wr, 294 op/s 2024-04-26T04:58:59.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:58:59 smithi005 ceph-mon[95809]: osdmap e730: 8 total, 8 up, 8 in 2024-04-26T04:59:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:00 smithi111 ceph-mon[86360]: osdmap e731: 8 total, 8 up, 8 in 2024-04-26T04:59:00.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:00 smithi005 ceph-mon[91801]: osdmap e731: 8 total, 8 up, 8 in 2024-04-26T04:59:00.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:00 smithi005 ceph-mon[95809]: osdmap e731: 8 total, 8 up, 8 in 2024-04-26T04:59:01.044 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (1984 ms) 2024-04-26T04:59:01.045 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-04-26T04:59:01.045 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:01.109 INFO:tasks.workunit.client.0.smithi005.stdout:1: acquiring exclusive lock 2024-04-26T04:59:01.109 INFO:tasks.workunit.client.0.smithi005.stdout:2: acquiring exclusive lock 2024-04-26T04:59:01.113 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock acquired 2024-04-26T04:59:01.138 INFO:tasks.workunit.client.0.smithi005.stdout:2: releasing exclusive lock 2024-04-26T04:59:01.140 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock released 2024-04-26T04:59:01.140 INFO:tasks.workunit.client.0.smithi005.stdout:2: acquiring exclusive lock 2024-04-26T04:59:01.144 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock acquired 2024-04-26T04:59:01.163 INFO:tasks.workunit.client.0.smithi005.stdout:2: releasing exclusive lock 2024-04-26T04:59:01.166 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock released 2024-04-26T04:59:01.166 INFO:tasks.workunit.client.0.smithi005.stdout:2: acquiring exclusive lock 2024-04-26T04:59:01.170 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock acquired 2024-04-26T04:59:01.203 INFO:tasks.workunit.client.0.smithi005.stdout:1: releasing exclusive lock 2024-04-26T04:59:01.205 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock released 2024-04-26T04:59:01.205 INFO:tasks.workunit.client.0.smithi005.stdout:1: acquiring exclusive lock 2024-04-26T04:59:01.208 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock acquired 2024-04-26T04:59:01.232 INFO:tasks.workunit.client.0.smithi005.stdout:2: releasing exclusive lock 2024-04-26T04:59:01.235 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock released 2024-04-26T04:59:01.235 INFO:tasks.workunit.client.0.smithi005.stdout:2: acquiring exclusive lock 2024-04-26T04:59:01.238 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock acquired 2024-04-26T04:59:01.266 INFO:tasks.workunit.client.0.smithi005.stdout:1: releasing exclusive lock 2024-04-26T04:59:01.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:01 smithi111 ceph-mon[86360]: pgmap v766: 617 pgs: 617 active+clean; 147 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 271 KiB/s wr, 244 op/s 2024-04-26T04:59:01.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:01 smithi111 ceph-mon[86360]: osdmap e732: 8 total, 8 up, 8 in 2024-04-26T04:59:01.269 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock released 2024-04-26T04:59:01.269 INFO:tasks.workunit.client.0.smithi005.stdout:1: acquiring exclusive lock 2024-04-26T04:59:01.272 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock acquired 2024-04-26T04:59:01.298 INFO:tasks.workunit.client.0.smithi005.stdout:2: releasing exclusive lock 2024-04-26T04:59:01.300 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock released 2024-04-26T04:59:01.301 INFO:tasks.workunit.client.0.smithi005.stdout:2: acquiring exclusive lock 2024-04-26T04:59:01.304 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock acquired 2024-04-26T04:59:01.347 INFO:tasks.workunit.client.0.smithi005.stdout:2: releasing exclusive lock 2024-04-26T04:59:01.349 INFO:tasks.workunit.client.0.smithi005.stdout:2: exclusive lock released 2024-04-26T04:59:01.353 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock acquired 2024-04-26T04:59:01.355 INFO:tasks.workunit.client.0.smithi005.stdout:1: releasing exclusive lock 2024-04-26T04:59:01.357 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock released 2024-04-26T04:59:01.357 INFO:tasks.workunit.client.0.smithi005.stdout:1: acquiring exclusive lock 2024-04-26T04:59:01.360 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock acquired 2024-04-26T04:59:01.374 INFO:tasks.workunit.client.0.smithi005.stdout:1: releasing exclusive lock 2024-04-26T04:59:01.377 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock released 2024-04-26T04:59:01.377 INFO:tasks.workunit.client.0.smithi005.stdout:1: acquiring exclusive lock 2024-04-26T04:59:01.380 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock acquired 2024-04-26T04:59:01.430 INFO:tasks.workunit.client.0.smithi005.stdout:1: releasing exclusive lock 2024-04-26T04:59:01.432 INFO:tasks.workunit.client.0.smithi005.stdout:1: exclusive lock released 2024-04-26T04:59:01.446 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ExclusiveLock (402 ms) 2024-04-26T04:59:01.446 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.BreakLock 2024-04-26T04:59:01.462 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:01.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:01 smithi005 ceph-mon[95809]: pgmap v766: 617 pgs: 617 active+clean; 147 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 271 KiB/s wr, 244 op/s 2024-04-26T04:59:01.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:01 smithi005 ceph-mon[95809]: osdmap e732: 8 total, 8 up, 8 in 2024-04-26T04:59:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:01 smithi005 ceph-mon[91801]: pgmap v766: 617 pgs: 617 active+clean; 147 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 271 KiB/s wr, 244 op/s 2024-04-26T04:59:01.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:01 smithi005 ceph-mon[91801]: osdmap e732: 8 total, 8 up, 8 in 2024-04-26T04:59:02.072 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.BreakLock (626 ms) 2024-04-26T04:59:02.072 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-04-26T04:59:02.072 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.098 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (26 ms) 2024-04-26T04:59:02.098 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-04-26T04:59:02.098 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-04-26T04:59:02.099 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-04-26T04:59:02.099 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.159 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:59:02.159 INFO:tasks.workunit.client.0.smithi005.stdout:1.56% done 2024-04-26T04:59:02.159 INFO:tasks.workunit.client.0.smithi005.stdout:3.12% done 2024-04-26T04:59:02.159 INFO:tasks.workunit.client.0.smithi005.stdout:4.69% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:6.25% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:7.81% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:9.38% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:10.94% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:12.50% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:14.06% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:15.62% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:17.19% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:18.75% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:20.31% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:21.88% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:23.44% done 2024-04-26T04:59:02.160 INFO:tasks.workunit.client.0.smithi005.stdout:25.00% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:26.56% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:28.12% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:29.69% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:31.25% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:32.81% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:34.38% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:35.94% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:37.50% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:39.06% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:40.62% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:42.19% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:43.75% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:45.31% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:46.88% done 2024-04-26T04:59:02.161 INFO:tasks.workunit.client.0.smithi005.stdout:48.44% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:50.00% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:51.56% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:53.12% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:54.69% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:56.25% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:57.81% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:59.38% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:60.94% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:62.50% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:64.06% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:65.62% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:67.19% done 2024-04-26T04:59:02.162 INFO:tasks.workunit.client.0.smithi005.stdout:68.75% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:70.31% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:71.88% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:73.44% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:75.00% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:76.56% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:78.12% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:79.69% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:81.25% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:82.81% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:84.38% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:85.94% done 2024-04-26T04:59:02.163 INFO:tasks.workunit.client.0.smithi005.stdout:87.50% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:89.06% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:90.62% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:92.19% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:93.75% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:95.31% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:96.88% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:98.44% done 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (61 ms) 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-04-26T04:59:02.164 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.215 INFO:tasks.workunit.client.0.smithi005.stdout:0.00% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:1.56% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:3.12% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:4.69% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:6.25% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:7.81% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:9.38% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:10.94% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:12.50% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:14.06% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:15.62% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:17.19% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:18.75% done 2024-04-26T04:59:02.216 INFO:tasks.workunit.client.0.smithi005.stdout:20.31% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:21.88% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:23.44% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:25.00% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:26.56% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:28.12% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:29.69% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:31.25% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:32.81% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:34.38% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:35.94% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:37.50% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:39.06% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:40.62% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:42.19% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:43.75% done 2024-04-26T04:59:02.217 INFO:tasks.workunit.client.0.smithi005.stdout:45.31% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:46.88% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:48.44% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:50.00% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:51.56% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:53.12% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:54.69% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:56.25% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:57.81% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:59.38% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:60.94% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:62.50% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:64.06% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:65.62% done 2024-04-26T04:59:02.218 INFO:tasks.workunit.client.0.smithi005.stdout:67.19% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:68.75% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:70.31% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:71.88% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:73.44% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:75.00% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:76.56% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:78.12% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:79.69% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:81.25% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:82.81% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:84.38% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:85.94% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:87.50% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:89.06% done 2024-04-26T04:59:02.219 INFO:tasks.workunit.client.0.smithi005.stdout:90.62% done 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:92.19% done 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:93.75% done 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:95.31% done 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:96.88% done 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:98.44% done 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (56 ms) 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-04-26T04:59:02.220 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.223 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.333 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestTrashPurge (118 ms) 2024-04-26T04:59:02.333 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-04-26T04:59:02.333 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.375 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (43 ms) 2024-04-26T04:59:02.376 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-04-26T04:59:02.376 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.396 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestListWatchers (20 ms) 2024-04-26T04:59:02.396 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-04-26T04:59:02.396 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:02 smithi005 ceph-mon[91801]: osdmap e733: 8 total, 8 up, 8 in 2024-04-26T04:59:02.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:02 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/1045584742"}]: dispatch 2024-04-26T04:59:02.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:02 smithi005 ceph-mon[95809]: osdmap e733: 8 total, 8 up, 8 in 2024-04-26T04:59:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:02 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/1045584742"}]: dispatch 2024-04-26T04:59:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:02 smithi111 ceph-mon[86360]: osdmap e733: 8 total, 8 up, 8 in 2024-04-26T04:59:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:02 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/1045584742"}]: dispatch 2024-04-26T04:59:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:03.061 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestSetSnapById (665 ms) 2024-04-26T04:59:03.062 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Namespaces 2024-04-26T04:59:03.094 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Namespaces (33 ms) 2024-04-26T04:59:03.094 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-04-26T04:59:03.122 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:03.127 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:03.200 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.NamespacesPP (106 ms) 2024-04-26T04:59:03.201 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Migration 2024-04-26T04:59:03.201 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:03.201 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:03.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-mon[95809]: pgmap v769: 617 pgs: 617 active+clean; 148 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 133 KiB/s wr, 62 op/s 2024-04-26T04:59:03.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/1045584742"}]': finished 2024-04-26T04:59:03.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-mon[95809]: osdmap e734: 8 total, 8 up, 8 in 2024-04-26T04:59:03.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-mon[91801]: pgmap v769: 617 pgs: 617 active+clean; 148 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 133 KiB/s wr, 62 op/s 2024-04-26T04:59:03.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/1045584742"}]': finished 2024-04-26T04:59:03.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-mon[91801]: osdmap e734: 8 total, 8 up, 8 in 2024-04-26T04:59:03.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:59:03] "GET /metrics HTTP/1.1" 200 75092 "" "Prometheus/2.43.0" 2024-04-26T04:59:03.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:03 smithi111 ceph-mon[86360]: pgmap v769: 617 pgs: 617 active+clean; 148 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 133 KiB/s wr, 62 op/s 2024-04-26T04:59:03.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:03 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.5:0/1045584742"}]': finished 2024-04-26T04:59:03.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:03 smithi111 ceph-mon[86360]: osdmap e734: 8 total, 8 up, 8 in 2024-04-26T04:59:03.598 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Migration (398 ms) 2024-04-26T04:59:03.599 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-04-26T04:59:03.599 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:03.599 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:03.994 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.MigrationPP (396 ms) 2024-04-26T04:59:03.994 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-04-26T04:59:03.994 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:04.009 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (16 ms) 2024-04-26T04:59:04.010 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-04-26T04:59:04.010 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:04.025 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (15 ms) 2024-04-26T04:59:04.025 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-04-26T04:59:04.025 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:04.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:04 smithi005 ceph-mon[91801]: osdmap e735: 8 total, 8 up, 8 in 2024-04-26T04:59:04.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:04 smithi005 ceph-mon[95809]: osdmap e735: 8 total, 8 up, 8 in 2024-04-26T04:59:04.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:04 smithi111 ceph-mon[86360]: osdmap e735: 8 total, 8 up, 8 in 2024-04-26T04:59:05.154 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (1129 ms) 2024-04-26T04:59:05.154 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-04-26T04:59:05.185 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.PoolMetadata (31 ms) 2024-04-26T04:59:05.185 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-04-26T04:59:05.203 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.PoolMetadataPP (18 ms) 2024-04-26T04:59:05.203 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.Config 2024-04-26T04:59:05.208 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:05.248 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.Config (45 ms) 2024-04-26T04:59:05.248 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-04-26T04:59:05.252 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:05.287 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ConfigPP (39 ms) 2024-04-26T04:59:05.287 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-04-26T04:59:05.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-mon[91801]: pgmap v772: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 316 KiB/s rd, 149 KiB/s wr, 356 op/s 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-mon[95809]: pgmap v772: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 316 KiB/s rd, 149 KiB/s wr, 356 op/s 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:59:05] ENGINE Bus STOPPING 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:59:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:59:05] ENGINE Bus STOPPED 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:59:05] ENGINE Bus STARTING 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:59:05] ENGINE Serving on http://:::9283 2024-04-26T04:59:05.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:05 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:04:59:05] ENGINE Bus STARTED 2024-04-26T04:59:05.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:05 smithi111 ceph-mon[86360]: pgmap v772: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 316 KiB/s rd, 149 KiB/s wr, 356 op/s 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[91801]: osdmap e736: 8 total, 8 up, 8 in 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[95809]: osdmap e736: 8 total, 8 up, 8 in 2024-04-26T04:59:06.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:59:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:06.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:59:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:06 smithi111 ceph-mon[86360]: osdmap e736: 8 total, 8 up, 8 in 2024-04-26T04:59:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:06 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1354207453' entity='client.admin' 2024-04-26T04:59:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:06 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:59:07.227 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[91801]: pgmap v774: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 19 KiB/s wr, 333 op/s 2024-04-26T04:59:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[91801]: pgmap v775: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 254 KiB/s rd, 20 KiB/s wr, 359 op/s 2024-04-26T04:59:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[91801]: osdmap e737: 8 total, 8 up, 8 in 2024-04-26T04:59:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2011838336' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:07.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:07.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[95809]: pgmap v774: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 19 KiB/s wr, 333 op/s 2024-04-26T04:59:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[95809]: pgmap v775: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 254 KiB/s rd, 20 KiB/s wr, 359 op/s 2024-04-26T04:59:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[95809]: osdmap e737: 8 total, 8 up, 8 in 2024-04-26T04:59:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2011838336' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:07.481 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:07 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:07 smithi111 ceph-mon[86360]: pgmap v774: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 19 KiB/s wr, 333 op/s 2024-04-26T04:59:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:07 smithi111 ceph-mon[86360]: pgmap v775: 617 pgs: 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 254 KiB/s rd, 20 KiB/s wr, 359 op/s 2024-04-26T04:59:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:07 smithi111 ceph-mon[86360]: osdmap e737: 8 total, 8 up, 8 in 2024-04-26T04:59:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:07 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2011838336' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:07.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:07 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[91801]: pgmap v777: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 376 KiB/s rd, 21 KiB/s wr, 505 op/s 2024-04-26T04:59:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[91801]: osdmap e738: 8 total, 8 up, 8 in 2024-04-26T04:59:08.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[91801]: osdmap e739: 8 total, 8 up, 8 in 2024-04-26T04:59:08.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:59:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[95809]: pgmap v777: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 376 KiB/s rd, 21 KiB/s wr, 505 op/s 2024-04-26T04:59:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[95809]: osdmap e738: 8 total, 8 up, 8 in 2024-04-26T04:59:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[95809]: osdmap e739: 8 total, 8 up, 8 in 2024-04-26T04:59:08.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:08 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:59:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:08 smithi111 ceph-mon[86360]: pgmap v777: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 376 KiB/s rd, 21 KiB/s wr, 505 op/s 2024-04-26T04:59:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:08 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:08 smithi111 ceph-mon[86360]: osdmap e738: 8 total, 8 up, 8 in 2024-04-26T04:59:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:08 smithi111 ceph-mon[86360]: osdmap e739: 8 total, 8 up, 8 in 2024-04-26T04:59:08.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:08 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:59:09.774 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:09.783 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:09.791 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:10.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:09 smithi111 ceph-mon[86360]: osdmap e740: 8 total, 8 up, 8 in 2024-04-26T04:59:10.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:09 smithi005 ceph-mon[91801]: osdmap e740: 8 total, 8 up, 8 in 2024-04-26T04:59:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:09 smithi005 ceph-mon[95809]: osdmap e740: 8 total, 8 up, 8 in 2024-04-26T04:59:10.818 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.PoolStatsPP (5531 ms) 2024-04-26T04:59:10.818 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-04-26T04:59:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:10 smithi111 ceph-mon[86360]: pgmap v781: 649 pgs: 7 creating+activating, 20 unknown, 622 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 761 KiB/s rd, 46 KiB/s wr, 1.00k op/s 2024-04-26T04:59:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:10 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:59:11.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:10 smithi111 ceph-mon[86360]: osdmap e741: 8 total, 8 up, 8 in 2024-04-26T04:59:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:10 smithi005 ceph-mon[91801]: pgmap v781: 649 pgs: 7 creating+activating, 20 unknown, 622 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 761 KiB/s rd, 46 KiB/s wr, 1.00k op/s 2024-04-26T04:59:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:10 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:59:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:10 smithi005 ceph-mon[91801]: osdmap e741: 8 total, 8 up, 8 in 2024-04-26T04:59:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:10 smithi005 ceph-mon[95809]: pgmap v781: 649 pgs: 7 creating+activating, 20 unknown, 622 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 761 KiB/s rd, 46 KiB/s wr, 1.00k op/s 2024-04-26T04:59:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:10 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:59:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:10 smithi005 ceph-mon[95809]: osdmap e741: 8 total, 8 up, 8 in 2024-04-26T04:59:12.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:11 smithi111 ceph-mon[86360]: osdmap e742: 8 total, 8 up, 8 in 2024-04-26T04:59:12.130 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:11 smithi005 ceph-mon[91801]: osdmap e742: 8 total, 8 up, 8 in 2024-04-26T04:59:12.130 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:11 smithi005 ceph-mon[95809]: osdmap e742: 8 total, 8 up, 8 in 2024-04-26T04:59:12.757 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:12 smithi111 ceph-mon[86360]: pgmap v784: 649 pgs: 7 creating+activating, 642 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 385 KiB/s rd, 29 KiB/s wr, 503 op/s 2024-04-26T04:59:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:12 smithi111 ceph-mon[86360]: osdmap e743: 8 total, 8 up, 8 in 2024-04-26T04:59:13.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:12 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3576995200' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:13.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:12 smithi005 ceph-mon[91801]: pgmap v784: 649 pgs: 7 creating+activating, 642 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 385 KiB/s rd, 29 KiB/s wr, 503 op/s 2024-04-26T04:59:13.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:12 smithi005 ceph-mon[91801]: osdmap e743: 8 total, 8 up, 8 in 2024-04-26T04:59:13.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:12 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3576995200' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:12 smithi005 ceph-mon[95809]: pgmap v784: 649 pgs: 7 creating+activating, 642 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 385 KiB/s rd, 29 KiB/s wr, 503 op/s 2024-04-26T04:59:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:12 smithi005 ceph-mon[95809]: osdmap e743: 8 total, 8 up, 8 in 2024-04-26T04:59:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:12 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3576995200' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:59:13] "GET /metrics HTTP/1.1" 200 75092 "" "Prometheus/2.43.0" 2024-04-26T04:59:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:13 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3576995200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:14.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:13 smithi111 ceph-mon[86360]: osdmap e744: 8 total, 8 up, 8 in 2024-04-26T04:59:14.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:13 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3576995200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:13 smithi005 ceph-mon[91801]: osdmap e744: 8 total, 8 up, 8 in 2024-04-26T04:59:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:13 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3576995200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:13 smithi005 ceph-mon[95809]: osdmap e744: 8 total, 8 up, 8 in 2024-04-26T04:59:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:14 smithi111 ceph-mon[86360]: pgmap v787: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 KiB/s wr, 38 op/s 2024-04-26T04:59:15.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:14 smithi111 ceph-mon[86360]: osdmap e745: 8 total, 8 up, 8 in 2024-04-26T04:59:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:14 smithi005 ceph-mon[91801]: pgmap v787: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 KiB/s wr, 38 op/s 2024-04-26T04:59:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:14 smithi005 ceph-mon[91801]: osdmap e745: 8 total, 8 up, 8 in 2024-04-26T04:59:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:14 smithi005 ceph-mon[95809]: pgmap v787: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 KiB/s wr, 38 op/s 2024-04-26T04:59:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:14 smithi005 ceph-mon[95809]: osdmap e745: 8 total, 8 up, 8 in 2024-04-26T04:59:16.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:15 smithi111 ceph-mon[86360]: osdmap e746: 8 total, 8 up, 8 in 2024-04-26T04:59:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:15 smithi005 ceph-mon[91801]: osdmap e746: 8 total, 8 up, 8 in 2024-04-26T04:59:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:15 smithi005 ceph-mon[95809]: osdmap e746: 8 total, 8 up, 8 in 2024-04-26T04:59:16.919 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ImageSpec (6101 ms) 2024-04-26T04:59:16.919 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-04-26T04:59:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:16 smithi111 ceph-mon[86360]: pgmap v790: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2024-04-26T04:59:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:16 smithi111 ceph-mon[86360]: osdmap e747: 8 total, 8 up, 8 in 2024-04-26T04:59:17.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:16 smithi005 ceph-mon[91801]: pgmap v790: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2024-04-26T04:59:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:16 smithi005 ceph-mon[91801]: osdmap e747: 8 total, 8 up, 8 in 2024-04-26T04:59:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:16 smithi005 ceph-mon[95809]: pgmap v790: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2024-04-26T04:59:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:16 smithi005 ceph-mon[95809]: osdmap e747: 8 total, 8 up, 8 in 2024-04-26T04:59:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[91801]: osdmap e748: 8 total, 8 up, 8 in 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[91801]: osdmap e749: 8 total, 8 up, 8 in 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/112899556' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[95809]: osdmap e748: 8 total, 8 up, 8 in 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[95809]: osdmap e749: 8 total, 8 up, 8 in 2024-04-26T04:59:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/112899556' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:18.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:17 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:17 smithi111 ceph-mon[86360]: osdmap e748: 8 total, 8 up, 8 in 2024-04-26T04:59:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:17 smithi111 ceph-mon[86360]: osdmap e749: 8 total, 8 up, 8 in 2024-04-26T04:59:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:17 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/112899556' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:17 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:18.737 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:18 smithi005 ceph-mon[91801]: pgmap v793: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 3.0 KiB/s wr, 76 op/s 2024-04-26T04:59:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:18 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:18 smithi005 ceph-mon[91801]: osdmap e750: 8 total, 8 up, 8 in 2024-04-26T04:59:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:18 smithi005 ceph-mon[95809]: pgmap v793: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 3.0 KiB/s wr, 76 op/s 2024-04-26T04:59:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:18 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:18 smithi005 ceph-mon[95809]: osdmap e750: 8 total, 8 up, 8 in 2024-04-26T04:59:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:18 smithi111 ceph-mon[86360]: pgmap v793: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 3.0 KiB/s wr, 76 op/s 2024-04-26T04:59:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:18 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:18 smithi111 ceph-mon[86360]: osdmap e750: 8 total, 8 up, 8 in 2024-04-26T04:59:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:20 smithi111 ceph-mon[86360]: pgmap v796: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 5.0 KiB/s wr, 106 op/s 2024-04-26T04:59:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:20 smithi111 ceph-mon[86360]: osdmap e751: 8 total, 8 up, 8 in 2024-04-26T04:59:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:20 smithi005 ceph-mon[91801]: pgmap v796: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 5.0 KiB/s wr, 106 op/s 2024-04-26T04:59:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:20 smithi005 ceph-mon[91801]: osdmap e751: 8 total, 8 up, 8 in 2024-04-26T04:59:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:20 smithi005 ceph-mon[95809]: pgmap v796: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 5.0 KiB/s wr, 106 op/s 2024-04-26T04:59:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:20 smithi005 ceph-mon[95809]: osdmap e751: 8 total, 8 up, 8 in 2024-04-26T04:59:22.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:21 smithi111 ceph-mon[86360]: osdmap e752: 8 total, 8 up, 8 in 2024-04-26T04:59:22.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:21 smithi005 ceph-mon[91801]: osdmap e752: 8 total, 8 up, 8 in 2024-04-26T04:59:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:21 smithi005 ceph-mon[95809]: osdmap e752: 8 total, 8 up, 8 in 2024-04-26T04:59:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:22 smithi111 ceph-mon[86360]: pgmap v799: 713 pgs: 17 unknown, 696 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 3.7 KiB/s wr, 128 op/s 2024-04-26T04:59:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:22 smithi111 ceph-mon[86360]: osdmap e753: 8 total, 8 up, 8 in 2024-04-26T04:59:23.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:22 smithi111 ceph-mon[86360]: osdmap e754: 8 total, 8 up, 8 in 2024-04-26T04:59:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:22 smithi005 ceph-mon[91801]: pgmap v799: 713 pgs: 17 unknown, 696 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 3.7 KiB/s wr, 128 op/s 2024-04-26T04:59:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:22 smithi005 ceph-mon[91801]: osdmap e753: 8 total, 8 up, 8 in 2024-04-26T04:59:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:22 smithi005 ceph-mon[91801]: osdmap e754: 8 total, 8 up, 8 in 2024-04-26T04:59:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:22 smithi005 ceph-mon[95809]: pgmap v799: 713 pgs: 17 unknown, 696 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 3.7 KiB/s wr, 128 op/s 2024-04-26T04:59:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:22 smithi005 ceph-mon[95809]: osdmap e753: 8 total, 8 up, 8 in 2024-04-26T04:59:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:22 smithi005 ceph-mon[95809]: osdmap e754: 8 total, 8 up, 8 in 2024-04-26T04:59:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:59:23] "GET /metrics HTTP/1.1" 200 79869 "" "Prometheus/2.43.0" 2024-04-26T04:59:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:24 smithi111 ceph-mon[86360]: pgmap v802: 713 pgs: 713 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 3.2 KiB/s wr, 104 op/s 2024-04-26T04:59:25.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:24 smithi111 ceph-mon[86360]: osdmap e755: 8 total, 8 up, 8 in 2024-04-26T04:59:25.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:24 smithi005 ceph-mon[91801]: pgmap v802: 713 pgs: 713 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 3.2 KiB/s wr, 104 op/s 2024-04-26T04:59:25.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:24 smithi005 ceph-mon[91801]: osdmap e755: 8 total, 8 up, 8 in 2024-04-26T04:59:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:24 smithi005 ceph-mon[95809]: pgmap v802: 713 pgs: 713 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 3.2 KiB/s wr, 104 op/s 2024-04-26T04:59:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:24 smithi005 ceph-mon[95809]: osdmap e755: 8 total, 8 up, 8 in 2024-04-26T04:59:25.798 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 3 2024-04-26T04:59:25.798 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 3 2024-04-26T04:59:25.798 INFO:tasks.workunit.client.0.smithi005.stdout:child: image201 2024-04-26T04:59:25.798 INFO:tasks.workunit.client.0.smithi005.stdout:child: image203 2024-04-26T04:59:25.798 INFO:tasks.workunit.client.0.smithi005.stdout:child: image202 2024-04-26T04:59:25.798 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-15/image201/86b8b650abe9 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-34/image202/86b8ee3180a5 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:found child test-librbd-smithi005-77850-15/image203/86b812a555e0 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:num children is: 0 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:num snaps is: 0 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:expected: 0 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8879 ms) 2024-04-26T04:59:25.799 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-04-26T04:59:26.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:25 smithi111 ceph-mon[86360]: osdmap e756: 8 total, 8 up, 8 in 2024-04-26T04:59:26.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:25 smithi005 ceph-mon[91801]: osdmap e756: 8 total, 8 up, 8 in 2024-04-26T04:59:26.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:25 smithi005 ceph-mon[95809]: osdmap e756: 8 total, 8 up, 8 in 2024-04-26T04:59:27.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:26 smithi005 ceph-mon[91801]: pgmap v805: 681 pgs: 681 active+clean; 2.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:26 smithi005 ceph-mon[91801]: osdmap e757: 8 total, 8 up, 8 in 2024-04-26T04:59:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:26 smithi005 ceph-mon[95809]: pgmap v805: 681 pgs: 681 active+clean; 2.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:26 smithi005 ceph-mon[95809]: osdmap e757: 8 total, 8 up, 8 in 2024-04-26T04:59:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:26 smithi111 ceph-mon[86360]: pgmap v805: 681 pgs: 681 active+clean; 2.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:26 smithi111 ceph-mon[86360]: osdmap e757: 8 total, 8 up, 8 in 2024-04-26T04:59:27.776 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:28.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[91801]: osdmap e758: 8 total, 8 up, 8 in 2024-04-26T04:59:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2302302722' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[91801]: osdmap e759: 8 total, 8 up, 8 in 2024-04-26T04:59:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[95809]: osdmap e758: 8 total, 8 up, 8 in 2024-04-26T04:59:28.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2302302722' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:28.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:28.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:28.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:27 smithi005 ceph-mon[95809]: osdmap e759: 8 total, 8 up, 8 in 2024-04-26T04:59:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:27 smithi111 ceph-mon[86360]: osdmap e758: 8 total, 8 up, 8 in 2024-04-26T04:59:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:27 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2302302722' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:27 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:59:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:27 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:59:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:27 smithi111 ceph-mon[86360]: osdmap e759: 8 total, 8 up, 8 in 2024-04-26T04:59:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:28 smithi005 ceph-mon[91801]: pgmap v808: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:59:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:28 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:59:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:28 smithi005 ceph-mon[91801]: osdmap e760: 8 total, 8 up, 8 in 2024-04-26T04:59:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:28 smithi005 ceph-mon[95809]: pgmap v808: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:59:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:28 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:59:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:28 smithi005 ceph-mon[95809]: osdmap e760: 8 total, 8 up, 8 in 2024-04-26T04:59:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:28 smithi111 ceph-mon[86360]: pgmap v808: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:59:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:28 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:59:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:28 smithi111 ceph-mon[86360]: osdmap e760: 8 total, 8 up, 8 in 2024-04-26T04:59:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:29 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:59:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:29 smithi005 ceph-mon[91801]: osdmap e761: 8 total, 8 up, 8 in 2024-04-26T04:59:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:29 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:59:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:29 smithi005 ceph-mon[95809]: osdmap e761: 8 total, 8 up, 8 in 2024-04-26T04:59:30.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:29 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:59:30.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:29 smithi111 ceph-mon[86360]: osdmap e761: 8 total, 8 up, 8 in 2024-04-26T04:59:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:30 smithi005 ceph-mon[91801]: pgmap v811: 713 pgs: 14 creating+peering, 18 unknown, 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 8.5 KiB/s wr, 175 op/s 2024-04-26T04:59:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:30 smithi005 ceph-mon[95809]: pgmap v811: 713 pgs: 14 creating+peering, 18 unknown, 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 8.5 KiB/s wr, 175 op/s 2024-04-26T04:59:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:30 smithi111 ceph-mon[86360]: pgmap v811: 713 pgs: 14 creating+peering, 18 unknown, 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 8.5 KiB/s wr, 175 op/s 2024-04-26T04:59:32.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:31 smithi111 ceph-mon[86360]: osdmap e762: 8 total, 8 up, 8 in 2024-04-26T04:59:32.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:31 smithi005 ceph-mon[91801]: osdmap e762: 8 total, 8 up, 8 in 2024-04-26T04:59:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:31 smithi005 ceph-mon[95809]: osdmap e762: 8 total, 8 up, 8 in 2024-04-26T04:59:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:33.189 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:32 smithi111 ceph-mon[86360]: pgmap v814: 713 pgs: 14 creating+peering, 699 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 3.0 KiB/s wr, 74 op/s 2024-04-26T04:59:33.189 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:32 smithi111 ceph-mon[86360]: osdmap e763: 8 total, 8 up, 8 in 2024-04-26T04:59:33.189 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:32 smithi111 ceph-mon[86360]: osdmap e764: 8 total, 8 up, 8 in 2024-04-26T04:59:33.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:32 smithi005 ceph-mon[91801]: pgmap v814: 713 pgs: 14 creating+peering, 699 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 3.0 KiB/s wr, 74 op/s 2024-04-26T04:59:33.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:32 smithi005 ceph-mon[91801]: osdmap e763: 8 total, 8 up, 8 in 2024-04-26T04:59:33.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:32 smithi005 ceph-mon[91801]: osdmap e764: 8 total, 8 up, 8 in 2024-04-26T04:59:33.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:32 smithi005 ceph-mon[95809]: pgmap v814: 713 pgs: 14 creating+peering, 699 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 3.0 KiB/s wr, 74 op/s 2024-04-26T04:59:33.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:32 smithi005 ceph-mon[95809]: osdmap e763: 8 total, 8 up, 8 in 2024-04-26T04:59:33.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:32 smithi005 ceph-mon[95809]: osdmap e764: 8 total, 8 up, 8 in 2024-04-26T04:59:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:59:33] "GET /metrics HTTP/1.1" 200 82279 "" "Prometheus/2.43.0" 2024-04-26T04:59:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:34 smithi111 ceph-mon[86360]: pgmap v817: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.7 KiB/s wr, 19 op/s 2024-04-26T04:59:35.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:34 smithi111 ceph-mon[86360]: osdmap e765: 8 total, 8 up, 8 in 2024-04-26T04:59:35.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:34 smithi005 ceph-mon[91801]: pgmap v817: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.7 KiB/s wr, 19 op/s 2024-04-26T04:59:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:34 smithi005 ceph-mon[91801]: osdmap e765: 8 total, 8 up, 8 in 2024-04-26T04:59:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:34 smithi005 ceph-mon[95809]: pgmap v817: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.7 KiB/s wr, 19 op/s 2024-04-26T04:59:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:34 smithi005 ceph-mon[95809]: osdmap e765: 8 total, 8 up, 8 in 2024-04-26T04:59:35.836 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.QuiesceWatch (10038 ms) 2024-04-26T04:59:35.836 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-04-26T04:59:35.836 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:36.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:35 smithi111 ceph-mon[86360]: osdmap e766: 8 total, 8 up, 8 in 2024-04-26T04:59:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:35 smithi005 ceph-mon[91801]: osdmap e766: 8 total, 8 up, 8 in 2024-04-26T04:59:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:35 smithi005 ceph-mon[95809]: osdmap e766: 8 total, 8 up, 8 in 2024-04-26T04:59:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:36 smithi111 ceph-mon[86360]: pgmap v820: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-26T04:59:37.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:36 smithi111 ceph-mon[86360]: osdmap e767: 8 total, 8 up, 8 in 2024-04-26T04:59:37.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:36 smithi005 ceph-mon[91801]: pgmap v820: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-26T04:59:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:36 smithi005 ceph-mon[91801]: osdmap e767: 8 total, 8 up, 8 in 2024-04-26T04:59:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:36 smithi005 ceph-mon[95809]: pgmap v820: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-26T04:59:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:36 smithi005 ceph-mon[95809]: osdmap e767: 8 total, 8 up, 8 in 2024-04-26T04:59:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:37 smithi111 ceph-mon[86360]: osdmap e768: 8 total, 8 up, 8 in 2024-04-26T04:59:38.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:37 smithi111 ceph-mon[86360]: osdmap e769: 8 total, 8 up, 8 in 2024-04-26T04:59:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:37 smithi005 ceph-mon[91801]: osdmap e768: 8 total, 8 up, 8 in 2024-04-26T04:59:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:37 smithi005 ceph-mon[91801]: osdmap e769: 8 total, 8 up, 8 in 2024-04-26T04:59:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:37 smithi005 ceph-mon[95809]: osdmap e768: 8 total, 8 up, 8 in 2024-04-26T04:59:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:37 smithi005 ceph-mon[95809]: osdmap e769: 8 total, 8 up, 8 in 2024-04-26T04:59:39.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:38 smithi111 ceph-mon[86360]: pgmap v823: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 5.0 KiB/s wr, 95 op/s 2024-04-26T04:59:39.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:38 smithi005 ceph-mon[91801]: pgmap v823: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 5.0 KiB/s wr, 95 op/s 2024-04-26T04:59:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:38 smithi005 ceph-mon[95809]: pgmap v823: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 5.0 KiB/s wr, 95 op/s 2024-04-26T04:59:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:39 smithi111 ceph-mon[86360]: osdmap e770: 8 total, 8 up, 8 in 2024-04-26T04:59:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:39 smithi005 ceph-mon[91801]: osdmap e770: 8 total, 8 up, 8 in 2024-04-26T04:59:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:39 smithi005 ceph-mon[95809]: osdmap e770: 8 total, 8 up, 8 in 2024-04-26T04:59:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:40 smithi005 ceph-mon[91801]: pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 6.7 KiB/s wr, 108 op/s 2024-04-26T04:59:41.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:40 smithi005 ceph-mon[91801]: osdmap e771: 8 total, 8 up, 8 in 2024-04-26T04:59:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:40 smithi005 ceph-mon[95809]: pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 6.7 KiB/s wr, 108 op/s 2024-04-26T04:59:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:40 smithi005 ceph-mon[95809]: osdmap e771: 8 total, 8 up, 8 in 2024-04-26T04:59:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:40 smithi111 ceph-mon[86360]: pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 6.7 KiB/s wr, 108 op/s 2024-04-26T04:59:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:40 smithi111 ceph-mon[86360]: osdmap e771: 8 total, 8 up, 8 in 2024-04-26T04:59:41.894 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (6058 ms) 2024-04-26T04:59:41.895 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-04-26T04:59:41.895 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:42.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:41 smithi005 ceph-mon[91801]: osdmap e772: 8 total, 8 up, 8 in 2024-04-26T04:59:42.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:41 smithi005 ceph-mon[95809]: osdmap e772: 8 total, 8 up, 8 in 2024-04-26T04:59:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:41 smithi111 ceph-mon[86360]: osdmap e772: 8 total, 8 up, 8 in 2024-04-26T04:59:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[95809]: pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 77 op/s 2024-04-26T04:59:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[95809]: osdmap e773: 8 total, 8 up, 8 in 2024-04-26T04:59:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:59:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[95809]: osdmap e774: 8 total, 8 up, 8 in 2024-04-26T04:59:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[91801]: pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 77 op/s 2024-04-26T04:59:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[91801]: osdmap e773: 8 total, 8 up, 8 in 2024-04-26T04:59:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:59:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:42 smithi005 ceph-mon[91801]: osdmap e774: 8 total, 8 up, 8 in 2024-04-26T04:59:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:42 smithi111 ceph-mon[86360]: pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 77 op/s 2024-04-26T04:59:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:42 smithi111 ceph-mon[86360]: osdmap e773: 8 total, 8 up, 8 in 2024-04-26T04:59:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:42 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T04:59:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:42 smithi111 ceph-mon[86360]: osdmap e774: 8 total, 8 up, 8 in 2024-04-26T04:59:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:59:43] "GET /metrics HTTP/1.1" 200 82279 "" "Prometheus/2.43.0" 2024-04-26T04:59:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:44 smithi111 ceph-mon[86360]: pgmap v832: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 5.2 KiB/s wr, 93 op/s 2024-04-26T04:59:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:44 smithi111 ceph-mon[86360]: osdmap e775: 8 total, 8 up, 8 in 2024-04-26T04:59:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:44 smithi005 ceph-mon[91801]: pgmap v832: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 5.2 KiB/s wr, 93 op/s 2024-04-26T04:59:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:44 smithi005 ceph-mon[91801]: osdmap e775: 8 total, 8 up, 8 in 2024-04-26T04:59:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:44 smithi005 ceph-mon[95809]: pgmap v832: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 5.2 KiB/s wr, 93 op/s 2024-04-26T04:59:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:44 smithi005 ceph-mon[95809]: osdmap e775: 8 total, 8 up, 8 in 2024-04-26T04:59:46.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:45 smithi111 ceph-mon[86360]: osdmap e776: 8 total, 8 up, 8 in 2024-04-26T04:59:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:45 smithi005 ceph-mon[91801]: osdmap e776: 8 total, 8 up, 8 in 2024-04-26T04:59:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:45 smithi005 ceph-mon[95809]: osdmap e776: 8 total, 8 up, 8 in 2024-04-26T04:59:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:46 smithi111 ceph-mon[86360]: pgmap v835: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 KiB/s wr, 29 op/s 2024-04-26T04:59:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:46 smithi111 ceph-mon[86360]: osdmap e777: 8 total, 8 up, 8 in 2024-04-26T04:59:47.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:46 smithi005 ceph-mon[91801]: pgmap v835: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 KiB/s wr, 29 op/s 2024-04-26T04:59:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:46 smithi005 ceph-mon[91801]: osdmap e777: 8 total, 8 up, 8 in 2024-04-26T04:59:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:46 smithi005 ceph-mon[95809]: pgmap v835: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 KiB/s wr, 29 op/s 2024-04-26T04:59:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:46 smithi005 ceph-mon[95809]: osdmap e777: 8 total, 8 up, 8 in 2024-04-26T04:59:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:47.843 INFO:tasks.workunit.client.0.smithi005.stdout:10.00% done 2024-04-26T04:59:47.844 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.QuiesceWatchError (5949 ms) 2024-04-26T04:59:47.844 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-04-26T04:59:47.844 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T04:59:47.861 INFO:tasks.workunit.client.0.smithi005.stderr:test quiesce is not long enough to time out 2024-04-26T04:59:48.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:47 smithi111 ceph-mon[86360]: osdmap e778: 8 total, 8 up, 8 in 2024-04-26T04:59:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:47 smithi005 ceph-mon[91801]: osdmap e778: 8 total, 8 up, 8 in 2024-04-26T04:59:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:47 smithi005 ceph-mon[95809]: osdmap e778: 8 total, 8 up, 8 in 2024-04-26T04:59:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:48 smithi111 ceph-mon[86360]: pgmap v838: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.0 KiB/s wr, 40 op/s 2024-04-26T04:59:49.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:48 smithi111 ceph-mon[86360]: osdmap e779: 8 total, 8 up, 8 in 2024-04-26T04:59:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:48 smithi005 ceph-mon[95809]: pgmap v838: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.0 KiB/s wr, 40 op/s 2024-04-26T04:59:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:48 smithi005 ceph-mon[95809]: osdmap e779: 8 total, 8 up, 8 in 2024-04-26T04:59:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:48 smithi005 ceph-mon[91801]: pgmap v838: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.0 KiB/s wr, 40 op/s 2024-04-26T04:59:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:48 smithi005 ceph-mon[91801]: osdmap e779: 8 total, 8 up, 8 in 2024-04-26T04:59:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:50 smithi111 ceph-mon[86360]: pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 6.0 KiB/s wr, 95 op/s 2024-04-26T04:59:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:50 smithi005 ceph-mon[91801]: pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 6.0 KiB/s wr, 95 op/s 2024-04-26T04:59:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:50 smithi005 ceph-mon[95809]: pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 6.0 KiB/s wr, 95 op/s 2024-04-26T04:59:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:52 smithi111 ceph-mon[86360]: pgmap v841: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.7 KiB/s wr, 73 op/s 2024-04-26T04:59:53.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:52 smithi111 ceph-mon[86360]: osdmap e780: 8 total, 8 up, 8 in 2024-04-26T04:59:53.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:52 smithi005 ceph-mon[91801]: pgmap v841: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.7 KiB/s wr, 73 op/s 2024-04-26T04:59:53.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:52 smithi005 ceph-mon[91801]: osdmap e780: 8 total, 8 up, 8 in 2024-04-26T04:59:53.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:52 smithi005 ceph-mon[95809]: pgmap v841: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.7 KiB/s wr, 73 op/s 2024-04-26T04:59:53.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:52 smithi005 ceph-mon[95809]: osdmap e780: 8 total, 8 up, 8 in 2024-04-26T04:59:53.453 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 04:59:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:04:59:53] "GET /metrics HTTP/1.1" 200 82286 "" "Prometheus/2.43.0" 2024-04-26T04:59:55.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:54 smithi111 ceph-mon[86360]: pgmap v843: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-04-26T04:59:55.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:54 smithi005 ceph-mon[91801]: pgmap v843: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-04-26T04:59:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:54 smithi005 ceph-mon[95809]: pgmap v843: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-04-26T04:59:56.794 INFO:tasks.workunit.client.0.smithi005.stderr:test quiesce is timed out 2024-04-26T04:59:56.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 0 2024-04-26T04:59:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:56 smithi005 ceph-mon[91801]: pgmap v844: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.4 KiB/s wr, 57 op/s 2024-04-26T04:59:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:56 smithi005 ceph-mon[95809]: pgmap v844: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.4 KiB/s wr, 57 op/s 2024-04-26T04:59:57.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:56 smithi111 ceph-mon[86360]: pgmap v844: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.4 KiB/s wr, 57 op/s 2024-04-26T04:59:57.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 1 2024-04-26T04:59:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:57 smithi005 ceph-mon[91801]: osdmap e781: 8 total, 8 up, 8 in 2024-04-26T04:59:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:57 smithi005 ceph-mon[95809]: osdmap e781: 8 total, 8 up, 8 in 2024-04-26T04:59:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:57 smithi111 ceph-mon[86360]: osdmap e781: 8 total, 8 up, 8 in 2024-04-26T04:59:58.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 2 2024-04-26T04:59:59.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 04:59:58 smithi005 ceph-mon[91801]: pgmap v846: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 1023 B/s wr, 24 op/s 2024-04-26T04:59:59.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 04:59:58 smithi005 ceph-mon[95809]: pgmap v846: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 1023 B/s wr, 24 op/s 2024-04-26T04:59:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 04:59:58 smithi111 ceph-mon[86360]: pgmap v846: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 1023 B/s wr, 24 op/s 2024-04-26T04:59:59.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 3 2024-04-26T05:00:00.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 4 2024-04-26T05:00:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:00 smithi005 ceph-mon[91801]: pgmap v847: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.1 KiB/s wr, 27 op/s 2024-04-26T05:00:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:00 smithi005 ceph-mon[91801]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:00 smithi005 ceph-mon[95809]: pgmap v847: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.1 KiB/s wr, 27 op/s 2024-04-26T05:00:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:00 smithi005 ceph-mon[95809]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:00 smithi111 ceph-mon[86360]: pgmap v847: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.1 KiB/s wr, 27 op/s 2024-04-26T05:00:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:00 smithi111 ceph-mon[86360]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 5 2024-04-26T05:00:02.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:02.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:02.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 6 2024-04-26T05:00:03.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:02 smithi005 ceph-mon[91801]: pgmap v848: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 KiB/s rd, 361 B/s wr, 4 op/s 2024-04-26T05:00:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:02 smithi005 ceph-mon[91801]: osdmap e782: 8 total, 8 up, 8 in 2024-04-26T05:00:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:02 smithi005 ceph-mon[95809]: pgmap v848: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 KiB/s rd, 361 B/s wr, 4 op/s 2024-04-26T05:00:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:02 smithi005 ceph-mon[95809]: osdmap e782: 8 total, 8 up, 8 in 2024-04-26T05:00:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:02 smithi111 ceph-mon[86360]: pgmap v848: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 KiB/s rd, 361 B/s wr, 4 op/s 2024-04-26T05:00:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:02 smithi111 ceph-mon[86360]: osdmap e782: 8 total, 8 up, 8 in 2024-04-26T05:00:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:00:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:00:03] "GET /metrics HTTP/1.1" 200 82287 "" "Prometheus/2.43.0" 2024-04-26T05:00:03.798 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 7 2024-04-26T05:00:04.797 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 8 2024-04-26T05:00:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: pgmap v850: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-04-26T05:00:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]: dispatch 2024-04-26T05:00:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]: dispatch 2024-04-26T05:00:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]: dispatch 2024-04-26T05:00:05.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: pgmap v850: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.232 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.233 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:04 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: pgmap v850: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]: dispatch 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]: dispatch 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-04-26T05:00:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:04 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]: dispatch 2024-04-26T05:00:05.798 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 9 2024-04-26T05:00:06.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]': finished 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: osdmap e783: 8 total, 8 up, 8 in 2024-04-26T05:00:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]': finished 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: osdmap e783: 8 total, 8 up, 8 in 2024-04-26T05:00:06.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:05 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.9", "id": [6, 3]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1d", "id": [0, 3]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1b", "id": [3, 1]}]': finished 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: osdmap e783: 8 total, 8 up, 8 in 2024-04-26T05:00:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:05 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:06.798 INFO:tasks.workunit.client.0.smithi005.stderr:waiting for timed out ... 10 2024-04-26T05:00:06.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[91801]: pgmap v852: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[91801]: osdmap e784: 8 total, 8 up, 8 in 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[95809]: pgmap v852: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[95809]: osdmap e784: 8 total, 8 up, 8 in 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:06.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:06 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:06 smithi111 ceph-mon[86360]: pgmap v852: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-04-26T05:00:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:06 smithi111 ceph-mon[86360]: osdmap e784: 8 total, 8 up, 8 in 2024-04-26T05:00:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:07.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:06 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:08.167 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:07 smithi005 ceph-mon[91801]: Health check failed: Degraded data redundancy: 1/531 objects degraded (0.188%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:00:08.167 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:07 smithi005 ceph-mon[95809]: Health check failed: Degraded data redundancy: 1/531 objects degraded (0.188%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:00:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:07 smithi111 ceph-mon[86360]: Health check failed: Degraded data redundancy: 1/531 objects degraded (0.188%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:00:08.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:08 smithi005 ceph-mon[91801]: pgmap v854: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 0 B/s, 0 objects/s recovering 2024-04-26T05:00:08.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:08 smithi005 ceph-mon[95809]: pgmap v854: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 0 B/s, 0 objects/s recovering 2024-04-26T05:00:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:08 smithi111 ceph-mon[86360]: pgmap v854: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 0 B/s, 0 objects/s recovering 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: pgmap v855: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 5 B/s, 0 objects/s recovering 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: pgmap v855: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 5 B/s, 0 objects/s recovering 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:11.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:10 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: pgmap v855: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 5 B/s, 0 objects/s recovering 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:10 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:12.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:11 smithi005 ceph-mon[91801]: pgmap v856: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 4 B/s, 0 objects/s recovering 2024-04-26T05:00:12.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:11 smithi005 ceph-mon[91801]: pgmap v857: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 6 B/s, 0 objects/s recovering 2024-04-26T05:00:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:11 smithi005 ceph-mon[95809]: pgmap v856: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 4 B/s, 0 objects/s recovering 2024-04-26T05:00:12.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:11 smithi005 ceph-mon[95809]: pgmap v857: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 6 B/s, 0 objects/s recovering 2024-04-26T05:00:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:11 smithi111 ceph-mon[86360]: pgmap v856: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 4 B/s, 0 objects/s recovering 2024-04-26T05:00:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:11 smithi111 ceph-mon[86360]: pgmap v857: 713 pgs: 1 activating+degraded, 3 activating, 1 peering, 708 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1/531 objects degraded (0.188%); 6 B/s, 0 objects/s recovering 2024-04-26T05:00:13.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:12 smithi005 ceph-mon[91801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/531 objects degraded (0.188%), 1 pg degraded) 2024-04-26T05:00:13.197 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:12 smithi005 ceph-mon[95809]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/531 objects degraded (0.188%), 1 pg degraded) 2024-04-26T05:00:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:12 smithi111 ceph-mon[86360]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/531 objects degraded (0.188%), 1 pg degraded) 2024-04-26T05:00:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:00:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:00:13] "GET /metrics HTTP/1.1" 200 82287 "" "Prometheus/2.43.0" 2024-04-26T05:00:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:13 smithi005 ceph-mon[91801]: pgmap v858: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2024-04-26T05:00:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:13 smithi005 ceph-mon[95809]: pgmap v858: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2024-04-26T05:00:14.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:13 smithi111 ceph-mon[86360]: pgmap v858: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2024-04-26T05:00:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:15 smithi005 ceph-mon[91801]: pgmap v859: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2024-04-26T05:00:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:15 smithi005 ceph-mon[95809]: pgmap v859: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2024-04-26T05:00:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:15 smithi111 ceph-mon[86360]: pgmap v859: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2024-04-26T05:00:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:16 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:16 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:16 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:00:17.798 INFO:tasks.workunit.client.0.smithi005.stderr:test retry succeeds 2024-04-26T05:00:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:17 smithi005 ceph-mon[95809]: pgmap v860: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3 B/s, 0 objects/s recovering 2024-04-26T05:00:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:17 smithi005 ceph-mon[91801]: pgmap v860: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3 B/s, 0 objects/s recovering 2024-04-26T05:00:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:17 smithi111 ceph-mon[86360]: pgmap v860: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3 B/s, 0 objects/s recovering 2024-04-26T05:00:19.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:18 smithi005 ceph-mon[91801]: osdmap e785: 8 total, 8 up, 8 in 2024-04-26T05:00:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:18 smithi005 ceph-mon[95809]: osdmap e785: 8 total, 8 up, 8 in 2024-04-26T05:00:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:18 smithi111 ceph-mon[86360]: osdmap e785: 8 total, 8 up, 8 in 2024-04-26T05:00:20.066 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32223 ms) 2024-04-26T05:00:20.066 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-04-26T05:00:20.066 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:00:20.125 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.WriteZeroes (59 ms) 2024-04-26T05:00:20.125 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-04-26T05:00:20.125 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:00:20.188 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (63 ms) 2024-04-26T05:00:20.188 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-04-26T05:00:20.188 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:00:20.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:19 smithi005 ceph-mon[95809]: pgmap v862: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-26T05:00:20.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:19 smithi005 ceph-mon[95809]: osdmap e786: 8 total, 8 up, 8 in 2024-04-26T05:00:20.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:19 smithi005 ceph-mon[91801]: pgmap v862: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-26T05:00:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:19 smithi005 ceph-mon[91801]: osdmap e786: 8 total, 8 up, 8 in 2024-04-26T05:00:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:19 smithi111 ceph-mon[86360]: pgmap v862: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-26T05:00:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:19 smithi111 ceph-mon[86360]: osdmap e786: 8 total, 8 up, 8 in 2024-04-26T05:00:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:20 smithi005 ceph-mon[91801]: osdmap e787: 8 total, 8 up, 8 in 2024-04-26T05:00:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:20 smithi005 ceph-mon[95809]: osdmap e787: 8 total, 8 up, 8 in 2024-04-26T05:00:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:20 smithi111 ceph-mon[86360]: osdmap e787: 8 total, 8 up, 8 in 2024-04-26T05:00:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:21 smithi005 ceph-mon[95809]: pgmap v865: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:21 smithi005 ceph-mon[95809]: osdmap e788: 8 total, 8 up, 8 in 2024-04-26T05:00:22.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:21 smithi005 ceph-mon[91801]: pgmap v865: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:22.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:21 smithi005 ceph-mon[91801]: osdmap e788: 8 total, 8 up, 8 in 2024-04-26T05:00:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:21 smithi111 ceph-mon[86360]: pgmap v865: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:21 smithi111 ceph-mon[86360]: osdmap e788: 8 total, 8 up, 8 in 2024-04-26T05:00:23.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:22 smithi005 ceph-mon[91801]: osdmap e789: 8 total, 8 up, 8 in 2024-04-26T05:00:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:22 smithi005 ceph-mon[95809]: osdmap e789: 8 total, 8 up, 8 in 2024-04-26T05:00:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:22 smithi111 ceph-mon[86360]: osdmap e789: 8 total, 8 up, 8 in 2024-04-26T05:00:23.463 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:00:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:00:23] "GET /metrics HTTP/1.1" 200 82291 "" "Prometheus/2.43.0" 2024-04-26T05:00:24.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:23 smithi005 ceph-mon[91801]: pgmap v868: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.2 KiB/s wr, 31 op/s 2024-04-26T05:00:24.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:23 smithi005 ceph-mon[91801]: osdmap e790: 8 total, 8 up, 8 in 2024-04-26T05:00:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:23 smithi005 ceph-mon[95809]: pgmap v868: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.2 KiB/s wr, 31 op/s 2024-04-26T05:00:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:23 smithi005 ceph-mon[95809]: osdmap e790: 8 total, 8 up, 8 in 2024-04-26T05:00:24.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:23 smithi111 ceph-mon[86360]: pgmap v868: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.2 KiB/s wr, 31 op/s 2024-04-26T05:00:24.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:23 smithi111 ceph-mon[86360]: osdmap e790: 8 total, 8 up, 8 in 2024-04-26T05:00:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:24 smithi111 ceph-mon[86360]: osdmap e791: 8 total, 8 up, 8 in 2024-04-26T05:00:25.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:24 smithi005 ceph-mon[91801]: osdmap e791: 8 total, 8 up, 8 in 2024-04-26T05:00:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:24 smithi005 ceph-mon[95809]: osdmap e791: 8 total, 8 up, 8 in 2024-04-26T05:00:26.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:25 smithi111 ceph-mon[86360]: pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 128 KiB/s rd, 10 KiB/s wr, 180 op/s 2024-04-26T05:00:26.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:25 smithi111 ceph-mon[86360]: osdmap e792: 8 total, 8 up, 8 in 2024-04-26T05:00:26.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:25 smithi005 ceph-mon[95809]: pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 128 KiB/s rd, 10 KiB/s wr, 180 op/s 2024-04-26T05:00:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:25 smithi005 ceph-mon[95809]: osdmap e792: 8 total, 8 up, 8 in 2024-04-26T05:00:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:25 smithi005 ceph-mon[91801]: pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 128 KiB/s rd, 10 KiB/s wr, 180 op/s 2024-04-26T05:00:26.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:25 smithi005 ceph-mon[91801]: osdmap e792: 8 total, 8 up, 8 in 2024-04-26T05:00:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:27 smithi111 ceph-mon[86360]: osdmap e793: 8 total, 8 up, 8 in 2024-04-26T05:00:27.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:27 smithi005 ceph-mon[91801]: osdmap e793: 8 total, 8 up, 8 in 2024-04-26T05:00:27.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:27 smithi005 ceph-mon[95809]: osdmap e793: 8 total, 8 up, 8 in 2024-04-26T05:00:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:28 smithi111 ceph-mon[86360]: pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 155 op/s 2024-04-26T05:00:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:28 smithi111 ceph-mon[86360]: osdmap e794: 8 total, 8 up, 8 in 2024-04-26T05:00:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:28 smithi111 ceph-mon[86360]: osdmap e795: 8 total, 8 up, 8 in 2024-04-26T05:00:28.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:28 smithi005 ceph-mon[91801]: pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 155 op/s 2024-04-26T05:00:28.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:28 smithi005 ceph-mon[91801]: osdmap e794: 8 total, 8 up, 8 in 2024-04-26T05:00:28.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:28 smithi005 ceph-mon[91801]: osdmap e795: 8 total, 8 up, 8 in 2024-04-26T05:00:28.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:28 smithi005 ceph-mon[95809]: pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 155 op/s 2024-04-26T05:00:28.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:28 smithi005 ceph-mon[95809]: osdmap e794: 8 total, 8 up, 8 in 2024-04-26T05:00:28.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:28 smithi005 ceph-mon[95809]: osdmap e795: 8 total, 8 up, 8 in 2024-04-26T05:00:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:29 smithi111 ceph-mon[86360]: pgmap v877: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 KiB/s wr, 15 op/s 2024-04-26T05:00:30.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:29 smithi111 ceph-mon[86360]: osdmap e796: 8 total, 8 up, 8 in 2024-04-26T05:00:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:29 smithi005 ceph-mon[91801]: pgmap v877: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 KiB/s wr, 15 op/s 2024-04-26T05:00:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:29 smithi005 ceph-mon[91801]: osdmap e796: 8 total, 8 up, 8 in 2024-04-26T05:00:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:29 smithi005 ceph-mon[95809]: pgmap v877: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 KiB/s wr, 15 op/s 2024-04-26T05:00:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:29 smithi005 ceph-mon[95809]: osdmap e796: 8 total, 8 up, 8 in 2024-04-26T05:00:31.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:30 smithi111 ceph-mon[86360]: osdmap e797: 8 total, 8 up, 8 in 2024-04-26T05:00:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:30 smithi005 ceph-mon[91801]: osdmap e797: 8 total, 8 up, 8 in 2024-04-26T05:00:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:30 smithi005 ceph-mon[95809]: osdmap e797: 8 total, 8 up, 8 in 2024-04-26T05:00:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:31 smithi111 ceph-mon[86360]: pgmap v880: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.5 KiB/s wr, 9 op/s 2024-04-26T05:00:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:31 smithi111 ceph-mon[86360]: osdmap e798: 8 total, 8 up, 8 in 2024-04-26T05:00:32.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:31 smithi005 ceph-mon[91801]: pgmap v880: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.5 KiB/s wr, 9 op/s 2024-04-26T05:00:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:31 smithi005 ceph-mon[91801]: osdmap e798: 8 total, 8 up, 8 in 2024-04-26T05:00:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:31 smithi005 ceph-mon[95809]: pgmap v880: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.5 KiB/s wr, 9 op/s 2024-04-26T05:00:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:31 smithi005 ceph-mon[95809]: osdmap e798: 8 total, 8 up, 8 in 2024-04-26T05:00:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:33.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:32 smithi005 ceph-mon[91801]: osdmap e799: 8 total, 8 up, 8 in 2024-04-26T05:00:33.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:32 smithi005 ceph-mon[95809]: osdmap e799: 8 total, 8 up, 8 in 2024-04-26T05:00:33.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:32 smithi111 ceph-mon[86360]: osdmap e799: 8 total, 8 up, 8 in 2024-04-26T05:00:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:00:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:00:33] "GET /metrics HTTP/1.1" 200 82292 "" "Prometheus/2.43.0" 2024-04-26T05:00:34.176 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:33 smithi111 ceph-mon[86360]: pgmap v883: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 1023 B/s wr, 6 op/s 2024-04-26T05:00:34.176 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:33 smithi111 ceph-mon[86360]: osdmap e800: 8 total, 8 up, 8 in 2024-04-26T05:00:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:33 smithi005 ceph-mon[91801]: pgmap v883: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 1023 B/s wr, 6 op/s 2024-04-26T05:00:34.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:33 smithi005 ceph-mon[91801]: osdmap e800: 8 total, 8 up, 8 in 2024-04-26T05:00:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:33 smithi005 ceph-mon[95809]: pgmap v883: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 1023 B/s wr, 6 op/s 2024-04-26T05:00:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:33 smithi005 ceph-mon[95809]: osdmap e800: 8 total, 8 up, 8 in 2024-04-26T05:00:35.114 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:34 smithi111 ceph-mon[86360]: osdmap e801: 8 total, 8 up, 8 in 2024-04-26T05:00:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:34 smithi005 ceph-mon[91801]: osdmap e801: 8 total, 8 up, 8 in 2024-04-26T05:00:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:34 smithi005 ceph-mon[95809]: osdmap e801: 8 total, 8 up, 8 in 2024-04-26T05:00:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:35 smithi005 ceph-mon[91801]: pgmap v886: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 5.5 KiB/s wr, 245 op/s 2024-04-26T05:00:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:35 smithi005 ceph-mon[91801]: osdmap e802: 8 total, 8 up, 8 in 2024-04-26T05:00:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:35 smithi005 ceph-mon[95809]: pgmap v886: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 5.5 KiB/s wr, 245 op/s 2024-04-26T05:00:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:35 smithi005 ceph-mon[95809]: osdmap e802: 8 total, 8 up, 8 in 2024-04-26T05:00:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:35 smithi111 ceph-mon[86360]: pgmap v886: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 5.5 KiB/s wr, 245 op/s 2024-04-26T05:00:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:35 smithi111 ceph-mon[86360]: osdmap e802: 8 total, 8 up, 8 in 2024-04-26T05:00:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:36 smithi005 ceph-mon[91801]: osdmap e803: 8 total, 8 up, 8 in 2024-04-26T05:00:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:36 smithi005 ceph-mon[95809]: osdmap e803: 8 total, 8 up, 8 in 2024-04-26T05:00:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:36 smithi111 ceph-mon[86360]: osdmap e803: 8 total, 8 up, 8 in 2024-04-26T05:00:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:37 smithi005 ceph-mon[91801]: pgmap v889: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.5 KiB/s wr, 243 op/s 2024-04-26T05:00:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:37 smithi005 ceph-mon[91801]: osdmap e804: 8 total, 8 up, 8 in 2024-04-26T05:00:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:37 smithi005 ceph-mon[91801]: osdmap e805: 8 total, 8 up, 8 in 2024-04-26T05:00:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:37 smithi005 ceph-mon[95809]: pgmap v889: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.5 KiB/s wr, 243 op/s 2024-04-26T05:00:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:37 smithi005 ceph-mon[95809]: osdmap e804: 8 total, 8 up, 8 in 2024-04-26T05:00:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:37 smithi005 ceph-mon[95809]: osdmap e805: 8 total, 8 up, 8 in 2024-04-26T05:00:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:37 smithi111 ceph-mon[86360]: pgmap v889: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.5 KiB/s wr, 243 op/s 2024-04-26T05:00:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:37 smithi111 ceph-mon[86360]: osdmap e804: 8 total, 8 up, 8 in 2024-04-26T05:00:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:37 smithi111 ceph-mon[86360]: osdmap e805: 8 total, 8 up, 8 in 2024-04-26T05:00:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:39 smithi111 ceph-mon[86360]: pgmap v892: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-04-26T05:00:40.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:39 smithi111 ceph-mon[86360]: osdmap e806: 8 total, 8 up, 8 in 2024-04-26T05:00:40.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:39 smithi005 ceph-mon[91801]: pgmap v892: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-04-26T05:00:40.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:39 smithi005 ceph-mon[91801]: osdmap e806: 8 total, 8 up, 8 in 2024-04-26T05:00:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:39 smithi005 ceph-mon[95809]: pgmap v892: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-04-26T05:00:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:39 smithi005 ceph-mon[95809]: osdmap e806: 8 total, 8 up, 8 in 2024-04-26T05:00:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:40 smithi111 ceph-mon[86360]: osdmap e807: 8 total, 8 up, 8 in 2024-04-26T05:00:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:40 smithi005 ceph-mon[91801]: osdmap e807: 8 total, 8 up, 8 in 2024-04-26T05:00:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:40 smithi005 ceph-mon[95809]: osdmap e807: 8 total, 8 up, 8 in 2024-04-26T05:00:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:41 smithi111 ceph-mon[86360]: pgmap v895: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.0 KiB/s wr, 19 op/s 2024-04-26T05:00:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:41 smithi111 ceph-mon[86360]: osdmap e808: 8 total, 8 up, 8 in 2024-04-26T05:00:42.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:41 smithi005 ceph-mon[91801]: pgmap v895: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.0 KiB/s wr, 19 op/s 2024-04-26T05:00:42.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:41 smithi005 ceph-mon[91801]: osdmap e808: 8 total, 8 up, 8 in 2024-04-26T05:00:42.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:41 smithi005 ceph-mon[95809]: pgmap v895: 713 pgs: 2 active+clean+snaptrim, 711 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.0 KiB/s wr, 19 op/s 2024-04-26T05:00:42.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:41 smithi005 ceph-mon[95809]: osdmap e808: 8 total, 8 up, 8 in 2024-04-26T05:00:43.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:42 smithi005 ceph-mon[95809]: osdmap e809: 8 total, 8 up, 8 in 2024-04-26T05:00:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:42 smithi005 ceph-mon[91801]: osdmap e809: 8 total, 8 up, 8 in 2024-04-26T05:00:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:42 smithi111 ceph-mon[86360]: osdmap e809: 8 total, 8 up, 8 in 2024-04-26T05:00:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:00:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:00:43] "GET /metrics HTTP/1.1" 200 82292 "" "Prometheus/2.43.0" 2024-04-26T05:00:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:43 smithi005 ceph-mon[95809]: pgmap v898: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-26T05:00:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:43 smithi005 ceph-mon[95809]: osdmap e810: 8 total, 8 up, 8 in 2024-04-26T05:00:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:43 smithi005 ceph-mon[91801]: pgmap v898: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-26T05:00:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:43 smithi005 ceph-mon[91801]: osdmap e810: 8 total, 8 up, 8 in 2024-04-26T05:00:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:43 smithi111 ceph-mon[86360]: pgmap v898: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 255 B/s wr, 7 op/s 2024-04-26T05:00:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:43 smithi111 ceph-mon[86360]: osdmap e810: 8 total, 8 up, 8 in 2024-04-26T05:00:45.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:44 smithi111 ceph-mon[86360]: osdmap e811: 8 total, 8 up, 8 in 2024-04-26T05:00:45.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:44 smithi005 ceph-mon[95809]: osdmap e811: 8 total, 8 up, 8 in 2024-04-26T05:00:45.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:44 smithi005 ceph-mon[91801]: osdmap e811: 8 total, 8 up, 8 in 2024-04-26T05:00:45.901 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] TestLibRBD.ConcurentOperations (25712 ms) 2024-04-26T05:00:46.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:45 smithi005 ceph-mon[95809]: pgmap v901: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 KiB/s wr, 53 op/s 2024-04-26T05:00:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:45 smithi005 ceph-mon[95809]: osdmap e812: 8 total, 8 up, 8 in 2024-04-26T05:00:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:45 smithi005 ceph-mon[91801]: pgmap v901: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 KiB/s wr, 53 op/s 2024-04-26T05:00:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:45 smithi005 ceph-mon[91801]: osdmap e812: 8 total, 8 up, 8 in 2024-04-26T05:00:46.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:45 smithi111 ceph-mon[86360]: pgmap v901: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 KiB/s wr, 53 op/s 2024-04-26T05:00:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:45 smithi111 ceph-mon[86360]: osdmap e812: 8 total, 8 up, 8 in 2024-04-26T05:00:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:46 smithi005 ceph-mon[91801]: osdmap e813: 8 total, 8 up, 8 in 2024-04-26T05:00:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:46 smithi005 ceph-mon[95809]: osdmap e813: 8 total, 8 up, 8 in 2024-04-26T05:00:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:46 smithi111 ceph-mon[86360]: osdmap e813: 8 total, 8 up, 8 in 2024-04-26T05:00:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:47 smithi005 ceph-mon[91801]: pgmap v904: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.7 KiB/s wr, 58 op/s 2024-04-26T05:00:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:47 smithi005 ceph-mon[91801]: osdmap e814: 8 total, 8 up, 8 in 2024-04-26T05:00:48.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:47 smithi005 ceph-mon[91801]: osdmap e815: 8 total, 8 up, 8 in 2024-04-26T05:00:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:47 smithi005 ceph-mon[95809]: pgmap v904: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.7 KiB/s wr, 58 op/s 2024-04-26T05:00:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:47 smithi005 ceph-mon[95809]: osdmap e814: 8 total, 8 up, 8 in 2024-04-26T05:00:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:47 smithi005 ceph-mon[95809]: osdmap e815: 8 total, 8 up, 8 in 2024-04-26T05:00:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:47 smithi111 ceph-mon[86360]: pgmap v904: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.7 KiB/s wr, 58 op/s 2024-04-26T05:00:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:47 smithi111 ceph-mon[86360]: osdmap e814: 8 total, 8 up, 8 in 2024-04-26T05:00:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:47 smithi111 ceph-mon[86360]: osdmap e815: 8 total, 8 up, 8 in 2024-04-26T05:00:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:49 smithi111 ceph-mon[86360]: pgmap v907: 649 pgs: 649 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 25 op/s 2024-04-26T05:00:50.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:49 smithi111 ceph-mon[86360]: osdmap e816: 8 total, 8 up, 8 in 2024-04-26T05:00:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:49 smithi005 ceph-mon[95809]: pgmap v907: 649 pgs: 649 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 25 op/s 2024-04-26T05:00:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:49 smithi005 ceph-mon[95809]: osdmap e816: 8 total, 8 up, 8 in 2024-04-26T05:00:50.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:49 smithi005 ceph-mon[91801]: pgmap v907: 649 pgs: 649 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 25 op/s 2024-04-26T05:00:50.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:49 smithi005 ceph-mon[91801]: osdmap e816: 8 total, 8 up, 8 in 2024-04-26T05:00:51.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:50 smithi111 ceph-mon[86360]: osdmap e817: 8 total, 8 up, 8 in 2024-04-26T05:00:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:50 smithi005 ceph-mon[91801]: osdmap e817: 8 total, 8 up, 8 in 2024-04-26T05:00:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:50 smithi005 ceph-mon[95809]: osdmap e817: 8 total, 8 up, 8 in 2024-04-26T05:00:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:51 smithi111 ceph-mon[86360]: pgmap v910: 585 pgs: 585 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:51 smithi111 ceph-mon[86360]: osdmap e818: 8 total, 8 up, 8 in 2024-04-26T05:00:52.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:51 smithi111 ceph-mon[86360]: osdmap e819: 8 total, 8 up, 8 in 2024-04-26T05:00:52.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:51 smithi005 ceph-mon[91801]: pgmap v910: 585 pgs: 585 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:51 smithi005 ceph-mon[91801]: osdmap e818: 8 total, 8 up, 8 in 2024-04-26T05:00:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:51 smithi005 ceph-mon[91801]: osdmap e819: 8 total, 8 up, 8 in 2024-04-26T05:00:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:51 smithi005 ceph-mon[95809]: pgmap v910: 585 pgs: 585 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:51 smithi005 ceph-mon[95809]: osdmap e818: 8 total, 8 up, 8 in 2024-04-26T05:00:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:51 smithi005 ceph-mon[95809]: osdmap e819: 8 total, 8 up, 8 in 2024-04-26T05:00:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:00:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:00:53] "GET /metrics HTTP/1.1" 200 82292 "" "Prometheus/2.43.0" 2024-04-26T05:00:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:53 smithi111 ceph-mon[86360]: pgmap v913: 521 pgs: 521 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:54.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:53 smithi111 ceph-mon[86360]: osdmap e820: 8 total, 8 up, 8 in 2024-04-26T05:00:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:53 smithi005 ceph-mon[95809]: pgmap v913: 521 pgs: 521 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:53 smithi005 ceph-mon[95809]: osdmap e820: 8 total, 8 up, 8 in 2024-04-26T05:00:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:53 smithi005 ceph-mon[91801]: pgmap v913: 521 pgs: 521 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:53 smithi005 ceph-mon[91801]: osdmap e820: 8 total, 8 up, 8 in 2024-04-26T05:00:55.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:54 smithi005 ceph-mon[91801]: osdmap e821: 8 total, 8 up, 8 in 2024-04-26T05:00:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:54 smithi005 ceph-mon[91801]: osdmap e822: 8 total, 8 up, 8 in 2024-04-26T05:00:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:54 smithi005 ceph-mon[95809]: osdmap e821: 8 total, 8 up, 8 in 2024-04-26T05:00:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:54 smithi005 ceph-mon[95809]: osdmap e822: 8 total, 8 up, 8 in 2024-04-26T05:00:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:54 smithi111 ceph-mon[86360]: osdmap e821: 8 total, 8 up, 8 in 2024-04-26T05:00:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:54 smithi111 ceph-mon[86360]: osdmap e822: 8 total, 8 up, 8 in 2024-04-26T05:00:55.778 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:55 smithi005 ceph-mon[91801]: pgmap v916: 457 pgs: 457 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:55.778 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:55 smithi005 ceph-mon[91801]: osdmap e823: 8 total, 8 up, 8 in 2024-04-26T05:00:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:55 smithi005 ceph-mon[95809]: pgmap v916: 457 pgs: 457 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:55 smithi005 ceph-mon[95809]: osdmap e823: 8 total, 8 up, 8 in 2024-04-26T05:00:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:55 smithi111 ceph-mon[86360]: pgmap v916: 457 pgs: 457 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:55 smithi111 ceph-mon[86360]: osdmap e823: 8 total, 8 up, 8 in 2024-04-26T05:00:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:57 smithi111 ceph-mon[86360]: pgmap v919: 393 pgs: 393 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:57 smithi111 ceph-mon[86360]: osdmap e824: 8 total, 8 up, 8 in 2024-04-26T05:00:58.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:57 smithi111 ceph-mon[86360]: osdmap e825: 8 total, 8 up, 8 in 2024-04-26T05:00:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:57 smithi005 ceph-mon[91801]: pgmap v919: 393 pgs: 393 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:57 smithi005 ceph-mon[91801]: osdmap e824: 8 total, 8 up, 8 in 2024-04-26T05:00:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:57 smithi005 ceph-mon[91801]: osdmap e825: 8 total, 8 up, 8 in 2024-04-26T05:00:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:57 smithi005 ceph-mon[95809]: pgmap v919: 393 pgs: 393 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:00:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:57 smithi005 ceph-mon[95809]: osdmap e824: 8 total, 8 up, 8 in 2024-04-26T05:00:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:57 smithi005 ceph-mon[95809]: osdmap e825: 8 total, 8 up, 8 in 2024-04-26T05:01:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:59 smithi111 ceph-mon[86360]: pgmap v922: 329 pgs: 329 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:00.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:00:59 smithi111 ceph-mon[86360]: osdmap e826: 8 total, 8 up, 8 in 2024-04-26T05:01:00.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:59 smithi005 ceph-mon[91801]: pgmap v922: 329 pgs: 329 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:00.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:00:59 smithi005 ceph-mon[91801]: osdmap e826: 8 total, 8 up, 8 in 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:59 smithi005 ceph-mon[95809]: pgmap v922: 329 pgs: 329 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:00:59 smithi005 ceph-mon[95809]: osdmap e826: 8 total, 8 up, 8 in 2024-04-26T05:01:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:00 smithi111 ceph-mon[86360]: osdmap e827: 8 total, 8 up, 8 in 2024-04-26T05:01:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:00 smithi005 ceph-mon[91801]: osdmap e827: 8 total, 8 up, 8 in 2024-04-26T05:01:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:00 smithi005 ceph-mon[95809]: osdmap e827: 8 total, 8 up, 8 in 2024-04-26T05:01:02.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:01 smithi005 ceph-mon[95809]: pgmap v925: 265 pgs: 265 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:01 smithi005 ceph-mon[95809]: osdmap e828: 8 total, 8 up, 8 in 2024-04-26T05:01:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:02.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:01 smithi005 ceph-mon[91801]: pgmap v925: 265 pgs: 265 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:02.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:01 smithi005 ceph-mon[91801]: osdmap e828: 8 total, 8 up, 8 in 2024-04-26T05:01:02.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:01 smithi111 ceph-mon[86360]: pgmap v925: 265 pgs: 265 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:01 smithi111 ceph-mon[86360]: osdmap e828: 8 total, 8 up, 8 in 2024-04-26T05:01:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:03.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:02 smithi005 ceph-mon[91801]: osdmap e829: 8 total, 8 up, 8 in 2024-04-26T05:01:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:02 smithi005 ceph-mon[95809]: osdmap e829: 8 total, 8 up, 8 in 2024-04-26T05:01:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:02 smithi111 ceph-mon[86360]: osdmap e829: 8 total, 8 up, 8 in 2024-04-26T05:01:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:01:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:01:03] "GET /metrics HTTP/1.1" 200 46268 "" "Prometheus/2.43.0" 2024-04-26T05:01:04.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:03 smithi005 ceph-mon[91801]: pgmap v928: 201 pgs: 201 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:04.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:03 smithi005 ceph-mon[91801]: osdmap e830: 8 total, 8 up, 8 in 2024-04-26T05:01:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:03 smithi005 ceph-mon[95809]: pgmap v928: 201 pgs: 201 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:04.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:03 smithi005 ceph-mon[95809]: osdmap e830: 8 total, 8 up, 8 in 2024-04-26T05:01:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:03 smithi111 ceph-mon[86360]: pgmap v928: 201 pgs: 201 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:03 smithi111 ceph-mon[86360]: osdmap e830: 8 total, 8 up, 8 in 2024-04-26T05:01:05.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:04 smithi005 ceph-mon[95809]: osdmap e831: 8 total, 8 up, 8 in 2024-04-26T05:01:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:04 smithi005 ceph-mon[91801]: osdmap e831: 8 total, 8 up, 8 in 2024-04-26T05:01:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:04 smithi111 ceph-mon[86360]: osdmap e831: 8 total, 8 up, 8 in 2024-04-26T05:01:05.790 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 159 tests from TestLibRBD (329617 ms total) 2024-04-26T05:01:05.791 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T05:01:05.791 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 15 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-04-26T05:01:05.824 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-04-26T05:01:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:05 smithi005 ceph-mon[91801]: pgmap v931: 137 pgs: 137 active+clean; 586 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:05 smithi005 ceph-mon[91801]: osdmap e832: 8 total, 8 up, 8 in 2024-04-26T05:01:06.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:05 smithi005 ceph-mon[91801]: osdmap e833: 8 total, 8 up, 8 in 2024-04-26T05:01:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:05 smithi005 ceph-mon[95809]: pgmap v931: 137 pgs: 137 active+clean; 586 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:05 smithi005 ceph-mon[95809]: osdmap e832: 8 total, 8 up, 8 in 2024-04-26T05:01:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:05 smithi005 ceph-mon[95809]: osdmap e833: 8 total, 8 up, 8 in 2024-04-26T05:01:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:05 smithi111 ceph-mon[86360]: pgmap v931: 137 pgs: 137 active+clean; 586 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:05 smithi111 ceph-mon[86360]: osdmap e832: 8 total, 8 up, 8 in 2024-04-26T05:01:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:05 smithi111 ceph-mon[86360]: osdmap e833: 8 total, 8 up, 8 in 2024-04-26T05:01:07.827 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:08.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:07 smithi005 ceph-mon[91801]: pgmap v934: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:07 smithi005 ceph-mon[91801]: osdmap e834: 8 total, 8 up, 8 in 2024-04-26T05:01:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:07 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/97266071' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:01:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:07 smithi005 ceph-mon[95809]: pgmap v934: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:07 smithi005 ceph-mon[95809]: osdmap e834: 8 total, 8 up, 8 in 2024-04-26T05:01:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:07 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/97266071' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:01:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:07 smithi111 ceph-mon[86360]: pgmap v934: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:07 smithi111 ceph-mon[86360]: osdmap e834: 8 total, 8 up, 8 in 2024-04-26T05:01:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:07 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/97266071' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:01:09.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:08 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/97266071' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T05:01:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:08 smithi005 ceph-mon[95809]: osdmap e835: 8 total, 8 up, 8 in 2024-04-26T05:01:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:08 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/97266071' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T05:01:09.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:08 smithi005 ceph-mon[91801]: osdmap e835: 8 total, 8 up, 8 in 2024-04-26T05:01:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:08 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/97266071' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T05:01:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:08 smithi111 ceph-mon[86360]: osdmap e835: 8 total, 8 up, 8 in 2024-04-26T05:01:09.852 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [3739832~5292,4232357~71908,5314473~73441,5764369~39937,6799221~70985,9565994~92837,10861209~58429,15278475~9388,20278864~27047] 2024-04-26T05:01:10.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:09 smithi005 ceph-mon[95809]: pgmap v937: 105 pgs: 15 creating+peering, 17 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:09 smithi005 ceph-mon[95809]: osdmap e836: 8 total, 8 up, 8 in 2024-04-26T05:01:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:09 smithi005 ceph-mon[91801]: pgmap v937: 105 pgs: 15 creating+peering, 17 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:10.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:09 smithi005 ceph-mon[91801]: osdmap e836: 8 total, 8 up, 8 in 2024-04-26T05:01:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:09 smithi111 ceph-mon[86360]: pgmap v937: 105 pgs: 15 creating+peering, 17 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:09 smithi111 ceph-mon[86360]: osdmap e836: 8 total, 8 up, 8 in 2024-04-26T05:01:10.838 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [3753955~69409,4132042~79877,5687758~39844,6305525~57610,8985210~42240,11251924~95304,12064144~72516,13788415~73874,17528389~51206,19748006~91628] 2024-04-26T05:01:10.847 INFO:tasks.workunit.client.0.smithi005.stdout: diff was [3745124~466795,5687758~39844,6305525~57610,8985210~42240,10919638~1217022,13788415~73874,17528389~51206,19748006~91628] 2024-04-26T05:01:10.850 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5026 ms) 2024-04-26T05:01:10.851 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministic 2024-04-26T05:01:10.851 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:11.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:10 smithi005 ceph-mon[95809]: osdmap e837: 8 total, 8 up, 8 in 2024-04-26T05:01:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:10 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:10 smithi005 ceph-mon[91801]: osdmap e837: 8 total, 8 up, 8 in 2024-04-26T05:01:11.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:10 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:10 smithi111 ceph-mon[86360]: osdmap e837: 8 total, 8 up, 8 in 2024-04-26T05:01:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:10 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:12.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:11 smithi111 ceph-mon[86360]: pgmap v940: 105 pgs: 15 creating+peering, 17 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:12.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:11 smithi111 ceph-mon[86360]: osdmap e838: 8 total, 8 up, 8 in 2024-04-26T05:01:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:11 smithi005 ceph-mon[91801]: pgmap v940: 105 pgs: 15 creating+peering, 17 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:11 smithi005 ceph-mon[91801]: osdmap e838: 8 total, 8 up, 8 in 2024-04-26T05:01:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:11 smithi005 ceph-mon[95809]: pgmap v940: 105 pgs: 15 creating+peering, 17 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:01:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:11 smithi005 ceph-mon[95809]: osdmap e838: 8 total, 8 up, 8 in 2024-04-26T05:01:12.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:12 smithi005 ceph-mon[91801]: osdmap e839: 8 total, 8 up, 8 in 2024-04-26T05:01:12.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:12 smithi005 ceph-mon[95809]: osdmap e839: 8 total, 8 up, 8 in 2024-04-26T05:01:13.237 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:12 smithi111 ceph-mon[86360]: osdmap e839: 8 total, 8 up, 8 in 2024-04-26T05:01:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:01:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:01:13] "GET /metrics HTTP/1.1" 200 46268 "" "Prometheus/2.43.0" 2024-04-26T05:01:13.864 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministic (3014 ms) 2024-04-26T05:01:13.864 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicPP 2024-04-26T05:01:13.864 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:14.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:13 smithi111 ceph-mon[86360]: pgmap v943: 105 pgs: 15 creating+peering, 90 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 142 KiB/s wr, 45 op/s 2024-04-26T05:01:14.163 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:13 smithi111 ceph-mon[86360]: osdmap e840: 8 total, 8 up, 8 in 2024-04-26T05:01:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:13 smithi005 ceph-mon[91801]: pgmap v943: 105 pgs: 15 creating+peering, 90 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 142 KiB/s wr, 45 op/s 2024-04-26T05:01:14.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:13 smithi005 ceph-mon[91801]: osdmap e840: 8 total, 8 up, 8 in 2024-04-26T05:01:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:13 smithi005 ceph-mon[95809]: pgmap v943: 105 pgs: 15 creating+peering, 90 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 142 KiB/s wr, 45 op/s 2024-04-26T05:01:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:13 smithi005 ceph-mon[95809]: osdmap e840: 8 total, 8 up, 8 in 2024-04-26T05:01:15.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[91801]: osdmap e841: 8 total, 8 up, 8 in 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[95809]: osdmap e841: 8 total, 8 up, 8 in 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:14 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:14 smithi111 ceph-mon[86360]: osdmap e841: 8 total, 8 up, 8 in 2024-04-26T05:01:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:14 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:14 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:14 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:14 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[91801]: pgmap v946: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 288 KiB/s wr, 68 op/s 2024-04-26T05:01:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[91801]: osdmap e842: 8 total, 8 up, 8 in 2024-04-26T05:01:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[95809]: pgmap v946: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 288 KiB/s wr, 68 op/s 2024-04-26T05:01:16.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[95809]: osdmap e842: 8 total, 8 up, 8 in 2024-04-26T05:01:16.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:16.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:16.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:15 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:15 smithi111 ceph-mon[86360]: pgmap v946: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 288 KiB/s wr, 68 op/s 2024-04-26T05:01:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:15 smithi111 ceph-mon[86360]: osdmap e842: 8 total, 8 up, 8 in 2024-04-26T05:01:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:15 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:16.885 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicPP (3021 ms) 2024-04-26T05:01:16.885 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-04-26T05:01:16.885 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[91801]: pgmap v948: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 172 KiB/s wr, 27 op/s 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[91801]: pgmap v949: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 216 KiB/s wr, 34 op/s 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[91801]: osdmap e843: 8 total, 8 up, 8 in 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[95809]: pgmap v948: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 172 KiB/s wr, 27 op/s 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[95809]: pgmap v949: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 216 KiB/s wr, 34 op/s 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[95809]: osdmap e843: 8 total, 8 up, 8 in 2024-04-26T05:01:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:16 smithi111 ceph-mon[86360]: pgmap v948: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 172 KiB/s wr, 27 op/s 2024-04-26T05:01:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:16 smithi111 ceph-mon[86360]: pgmap v949: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 216 KiB/s wr, 34 op/s 2024-04-26T05:01:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:16 smithi111 ceph-mon[86360]: osdmap e843: 8 total, 8 up, 8 in 2024-04-26T05:01:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:17 smithi005 ceph-mon[91801]: osdmap e844: 8 total, 8 up, 8 in 2024-04-26T05:01:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:17 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:17 smithi005 ceph-mon[91801]: osdmap e845: 8 total, 8 up, 8 in 2024-04-26T05:01:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:17 smithi005 ceph-mon[95809]: osdmap e844: 8 total, 8 up, 8 in 2024-04-26T05:01:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:17 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:17 smithi005 ceph-mon[95809]: osdmap e845: 8 total, 8 up, 8 in 2024-04-26T05:01:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:17 smithi111 ceph-mon[86360]: osdmap e844: 8 total, 8 up, 8 in 2024-04-26T05:01:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:17 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:01:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:17 smithi111 ceph-mon[86360]: osdmap e845: 8 total, 8 up, 8 in 2024-04-26T05:01:19.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:18 smithi005 ceph-mon[91801]: pgmap v952: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.6 KiB/s wr, 30 op/s 2024-04-26T05:01:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:18 smithi005 ceph-mon[91801]: osdmap e846: 8 total, 8 up, 8 in 2024-04-26T05:01:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:18 smithi005 ceph-mon[95809]: pgmap v952: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.6 KiB/s wr, 30 op/s 2024-04-26T05:01:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:18 smithi005 ceph-mon[95809]: osdmap e846: 8 total, 8 up, 8 in 2024-04-26T05:01:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:18 smithi111 ceph-mon[86360]: pgmap v952: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.6 KiB/s wr, 30 op/s 2024-04-26T05:01:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:18 smithi111 ceph-mon[86360]: osdmap e846: 8 total, 8 up, 8 in 2024-04-26T05:01:19.784 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (2899 ms) 2024-04-26T05:01:19.784 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTruncate 2024-04-26T05:01:19.784 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:20 smithi111 ceph-mon[86360]: pgmap v955: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 98 op/s 2024-04-26T05:01:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:20 smithi111 ceph-mon[86360]: osdmap e847: 8 total, 8 up, 8 in 2024-04-26T05:01:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:20 smithi005 ceph-mon[91801]: pgmap v955: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 98 op/s 2024-04-26T05:01:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:20 smithi005 ceph-mon[91801]: osdmap e847: 8 total, 8 up, 8 in 2024-04-26T05:01:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:20 smithi005 ceph-mon[95809]: pgmap v955: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 98 op/s 2024-04-26T05:01:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:20 smithi005 ceph-mon[95809]: osdmap e847: 8 total, 8 up, 8 in 2024-04-26T05:01:22.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:21 smithi005 ceph-mon[91801]: osdmap e848: 8 total, 8 up, 8 in 2024-04-26T05:01:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:21 smithi005 ceph-mon[95809]: osdmap e848: 8 total, 8 up, 8 in 2024-04-26T05:01:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:21 smithi111 ceph-mon[86360]: osdmap e848: 8 total, 8 up, 8 in 2024-04-26T05:01:22.818 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateTruncate (3033 ms) 2024-04-26T05:01:22.818 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateWriteAndTruncate 2024-04-26T05:01:22.818 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:23.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:22 smithi005 ceph-mon[91801]: pgmap v958: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 140 op/s 2024-04-26T05:01:23.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:22 smithi005 ceph-mon[91801]: osdmap e849: 8 total, 8 up, 8 in 2024-04-26T05:01:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:22 smithi005 ceph-mon[95809]: pgmap v958: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 140 op/s 2024-04-26T05:01:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:22 smithi005 ceph-mon[95809]: osdmap e849: 8 total, 8 up, 8 in 2024-04-26T05:01:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:22 smithi111 ceph-mon[86360]: pgmap v958: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 140 KiB/s wr, 140 op/s 2024-04-26T05:01:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:22 smithi111 ceph-mon[86360]: osdmap e849: 8 total, 8 up, 8 in 2024-04-26T05:01:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:01:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:01:23] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.43.0" 2024-04-26T05:01:24.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:23 smithi005 ceph-mon[91801]: osdmap e850: 8 total, 8 up, 8 in 2024-04-26T05:01:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:23 smithi005 ceph-mon[95809]: osdmap e850: 8 total, 8 up, 8 in 2024-04-26T05:01:24.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:23 smithi111 ceph-mon[86360]: osdmap e850: 8 total, 8 up, 8 in 2024-04-26T05:01:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:24 smithi005 ceph-mon[91801]: pgmap v961: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 138 KiB/s wr, 108 op/s 2024-04-26T05:01:24.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:24 smithi005 ceph-mon[91801]: osdmap e851: 8 total, 8 up, 8 in 2024-04-26T05:01:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:24 smithi005 ceph-mon[95809]: pgmap v961: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 138 KiB/s wr, 108 op/s 2024-04-26T05:01:24.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:24 smithi005 ceph-mon[95809]: osdmap e851: 8 total, 8 up, 8 in 2024-04-26T05:01:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:24 smithi111 ceph-mon[86360]: pgmap v961: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 138 KiB/s wr, 108 op/s 2024-04-26T05:01:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:24 smithi111 ceph-mon[86360]: osdmap e851: 8 total, 8 up, 8 in 2024-04-26T05:01:25.832 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateWriteAndTruncate (3014 ms) 2024-04-26T05:01:25.832 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-04-26T05:01:25.832 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:01:25.914 INFO:tasks.workunit.client.0.smithi005.stdout: i=0 exists [25602865~1622760,27326766~7845333,55397009~6597620,181493205~6937117,201878185~8272966,222194410~1972996,287138561~4778574,341536575~2725432] wrote [25602865~1622760,27326766~7845333,55397009~6597620,181493205~6937117,201878185~8272966,222194410~1972996,287138561~4778574,341536575~2725432] 2024-04-26T05:01:26.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:25 smithi005 ceph-mon[91801]: osdmap e852: 8 total, 8 up, 8 in 2024-04-26T05:01:26.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:25 smithi005 ceph-mon[95809]: osdmap e852: 8 total, 8 up, 8 in 2024-04-26T05:01:26.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:25 smithi111 ceph-mon[86360]: osdmap e852: 8 total, 8 up, 8 in 2024-04-26T05:01:26.890 INFO:tasks.workunit.client.0.smithi005.stdout: i=1 exists [25602865~1622760,27326766~9345451,55397009~6597620,84444570~14520168,137672071~4179285,181493205~7860165,192002560~3429953,200091691~1277945,201878185~8272966,222194410~1972996,238186251~6498645,287138561~4778574,341536575~2725432,363721045~8096572] wrote [29081750~7590467,56441850~5272261,84444570~14520168,137672071~4179285,187456785~1896585,192002560~3429953,200091691~1277945,238186251~6498645,363721045~8096572] 2024-04-26T05:01:27.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:26 smithi005 ceph-mon[91801]: pgmap v964: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.7 KiB/s wr, 46 op/s 2024-04-26T05:01:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:26 smithi005 ceph-mon[91801]: osdmap e853: 8 total, 8 up, 8 in 2024-04-26T05:01:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:26 smithi005 ceph-mon[95809]: pgmap v964: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.7 KiB/s wr, 46 op/s 2024-04-26T05:01:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:26 smithi005 ceph-mon[95809]: osdmap e853: 8 total, 8 up, 8 in 2024-04-26T05:01:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:26 smithi111 ceph-mon[86360]: pgmap v964: 105 pgs: 105 active+clean; 17 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.7 KiB/s wr, 46 op/s 2024-04-26T05:01:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:26 smithi111 ceph-mon[86360]: osdmap e853: 8 total, 8 up, 8 in 2024-04-26T05:01:27.873 INFO:tasks.workunit.client.0.smithi005.stdout: i=2 exists [25602865~1622760,27326766~9345451,44836370~8184528,55397009~6597620,84444570~14520168,132942611~8908745,157730584~617997,166051702~5657552,181493205~7860165,192002560~3429953,200091691~1277945,201878185~8272966,222194410~1972996,238186251~6498645,269611654~850468,287138561~4778574,300090207~6002606,327281779~6554677,341536575~2725432,360818268~10999349,375480946~4237330,394539851~1802255] wrote [44836370~8184528,132942611~6895907,157730584~617997,166051702~5657552,269611654~850468,300090207~6002606,327281779~6554677,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:28.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:27 smithi005 ceph-mon[91801]: osdmap e854: 8 total, 8 up, 8 in 2024-04-26T05:01:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:27 smithi005 ceph-mon[95809]: osdmap e854: 8 total, 8 up, 8 in 2024-04-26T05:01:28.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:27 smithi111 ceph-mon[86360]: osdmap e854: 8 total, 8 up, 8 in 2024-04-26T05:01:28.880 INFO:tasks.workunit.client.0.smithi005.stdout: i=3 exists [25602865~1622760,27326766~9345451,44836370~8184528,55397009~6597620,66421625~6334260,77060329~2914962,84444570~14520168,123388044~663063,132942611~8908745,143398206~3113874,157730584~617997,166051702~5657552,181493205~10364543,192002560~3429953,200091691~1277945,201878185~8272966,222194410~1972996,238186251~6498645,255108052~876750,269611654~850468,287138561~4778574,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,341536575~2725432,360818268~10999349,375480946~4237330,394539851~1802255] wrote [66421625~6334260,77060329~2914962,123388044~663063,143398206~3113874,188506823~3350925,255108052~876750,303888503~1788567,308835410~6979400,317187470~6538201] 2024-04-26T05:01:29.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:28 smithi005 ceph-mon[91801]: pgmap v967: 105 pgs: 105 active+clean; 42 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.0 MiB/s wr, 20 op/s 2024-04-26T05:01:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:28 smithi005 ceph-mon[91801]: osdmap e855: 8 total, 8 up, 8 in 2024-04-26T05:01:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:28 smithi005 ceph-mon[95809]: pgmap v967: 105 pgs: 105 active+clean; 42 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.0 MiB/s wr, 20 op/s 2024-04-26T05:01:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:28 smithi005 ceph-mon[95809]: osdmap e855: 8 total, 8 up, 8 in 2024-04-26T05:01:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:28 smithi111 ceph-mon[86360]: pgmap v967: 105 pgs: 105 active+clean; 42 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.0 MiB/s wr, 20 op/s 2024-04-26T05:01:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:28 smithi111 ceph-mon[86360]: osdmap e855: 8 total, 8 up, 8 in 2024-04-26T05:01:29.895 INFO:tasks.workunit.client.0.smithi005.stdout: i=4 exists [25602865~1622760,27326766~9345451,44836370~8184528,55397009~7810277,66421625~6334260,72793401~7181890,84444570~14520168,123388044~663063,132942611~8908745,143398206~3113874,151760462~7153854,166051702~5657552,181493205~10364543,192002560~9367076,201878185~8272966,214138291~4515375,222194410~1972996,226471898~6634796,238186251~8595190,255108052~876750,261080148~7565141,269611654~850468,287138561~4778574,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,341536575~2725432,346194868~2230218,360818268~10999349,375480946~4237330,394539851~1802255] wrote [61115753~2091533,72793401~4677649,151760462~7153854,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,261080148~7565141,346194868~2230218,364526075~58152] 2024-04-26T05:01:30.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:29 smithi005 ceph-mon[95809]: osdmap e856: 8 total, 8 up, 8 in 2024-04-26T05:01:30.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:29 smithi005 ceph-mon[91801]: osdmap e856: 8 total, 8 up, 8 in 2024-04-26T05:01:30.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:29 smithi111 ceph-mon[86360]: osdmap e856: 8 total, 8 up, 8 in 2024-04-26T05:01:30.898 INFO:tasks.workunit.client.0.smithi005.stdout: i=5 exists [25602865~1622760,27326766~9345451,44836370~8184528,55397009~7810277,66421625~6334260,72793401~7181890,81459847~22196,84444570~16588780,123388044~663063,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,181493205~10364543,192002560~9367076,201878185~8272966,214138291~4515375,222194410~1972996,226471898~6634796,238186251~8595190,255108052~876750,256469328~12175961,269611654~850468,287138561~4778574,300090207~6002606,308835410~6979400,317187470~7629265,327281779~6554677,335978829~661345,341536575~2725432,346194868~2230218,360818268~10999349,375480946~4237330,394539851~1802255] wrote [56614553~4963103,81459847~22196,93866620~7166730,147744643~3693647,197328753~1769922,256469328~6792010,320955023~3861712,335978829~661345,364459877~1335229] 2024-04-26T05:01:31.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:30 smithi005 ceph-mon[91801]: pgmap v970: 105 pgs: 105 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 12 MiB/s wr, 184 op/s 2024-04-26T05:01:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:30 smithi005 ceph-mon[91801]: osdmap e857: 8 total, 8 up, 8 in 2024-04-26T05:01:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:30 smithi005 ceph-mon[95809]: pgmap v970: 105 pgs: 105 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 12 MiB/s wr, 184 op/s 2024-04-26T05:01:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:30 smithi005 ceph-mon[95809]: osdmap e857: 8 total, 8 up, 8 in 2024-04-26T05:01:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:30 smithi111 ceph-mon[86360]: pgmap v970: 105 pgs: 105 active+clean; 80 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 12 MiB/s wr, 184 op/s 2024-04-26T05:01:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:30 smithi111 ceph-mon[86360]: osdmap e857: 8 total, 8 up, 8 in 2024-04-26T05:01:31.923 INFO:tasks.workunit.client.0.smithi005.stdout: i=6 exists [25602865~1622760,27326766~9345451,44836370~8184528,55397009~7810277,66421625~6334260,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,181493205~10364543,192002560~18148591,214138291~10029115,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,287138561~4778574,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,341536575~9288159,360818268~10999349,375480946~4237330,394539851~1802255] wrote [78964078~2902716,112016011~7352303,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,314923259~1047132,342966963~7857771] 2024-04-26T05:01:32.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:31 smithi111 ceph-mon[86360]: osdmap e858: 8 total, 8 up, 8 in 2024-04-26T05:01:32.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:31 smithi005 ceph-mon[91801]: osdmap e858: 8 total, 8 up, 8 in 2024-04-26T05:01:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:31 smithi005 ceph-mon[95809]: osdmap e858: 8 total, 8 up, 8 in 2024-04-26T05:01:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:32.904 INFO:tasks.workunit.client.0.smithi005.stdout: i=7 exists [9035947~8048222,25602865~1622760,27326766~9345451,44836370~8184528,55397009~7810277,66421625~6334260,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,181493205~10364543,192002560~18148591,214138291~10029115,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,287138561~4778574,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,394539851~3440498] wrote [9035947~8048222,125863971~6236078,273362189~2881329,283756946~2345559,318803882~1606752,338235024~5077544,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:32 smithi111 ceph-mon[86360]: pgmap v973: 105 pgs: 105 active+clean; 222 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 38 MiB/s wr, 179 op/s 2024-04-26T05:01:33.181 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:32 smithi111 ceph-mon[86360]: osdmap e859: 8 total, 8 up, 8 in 2024-04-26T05:01:33.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:32 smithi005 ceph-mon[91801]: pgmap v973: 105 pgs: 105 active+clean; 222 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 38 MiB/s wr, 179 op/s 2024-04-26T05:01:33.194 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:32 smithi005 ceph-mon[91801]: osdmap e859: 8 total, 8 up, 8 in 2024-04-26T05:01:33.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:32 smithi005 ceph-mon[95809]: pgmap v973: 105 pgs: 105 active+clean; 222 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 38 MiB/s wr, 179 op/s 2024-04-26T05:01:33.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:32 smithi005 ceph-mon[95809]: osdmap e859: 8 total, 8 up, 8 in 2024-04-26T05:01:33.482 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:01:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:01:33] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.43.0" 2024-04-26T05:01:33.902 INFO:tasks.workunit.client.0.smithi005.stdout: i=8 exists [4066564~13017605,22470476~1110648,25602865~1622760,27326766~12581856,44836370~8184528,55397009~17358876,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5657552,181493205~10364543,192002560~18148591,214138291~10029115,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,287138561~4778574,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,394539851~3440498] wrote [4066564~6157893,22470476~1110648,32116244~7792378,61261300~9016548,77367094~2471889,164569697~386756,199723975~6599185,397306655~47324] 2024-04-26T05:01:34.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:33 smithi005 ceph-mon[91801]: osdmap e860: 8 total, 8 up, 8 in 2024-04-26T05:01:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:33 smithi005 ceph-mon[95809]: osdmap e860: 8 total, 8 up, 8 in 2024-04-26T05:01:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:33 smithi111 ceph-mon[86360]: osdmap e860: 8 total, 8 up, 8 in 2024-04-26T05:01:34.924 INFO:tasks.workunit.client.0.smithi005.stdout: i=9 exists [4066564~17219964,22470476~1110648,25602865~1622760,27326766~12581856,44836370~8184528,55397009~17358876,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,181493205~28657946,214138291~10029115,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,287138561~4778574,300090207~6002606,308835410~7134981,317187470~9659816,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,382845155~4560594,394539851~3440498] wrote [10138532~11147996,60560864~6873295,96669846~4086612,166755333~5292336,191708486~7072079,323712297~3134989,382845155~4560594] 2024-04-26T05:01:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:34 smithi005 ceph-mon[95809]: pgmap v976: 105 pgs: 105 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 55 MiB/s wr, 286 op/s 2024-04-26T05:01:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:34 smithi005 ceph-mon[95809]: osdmap e861: 8 total, 8 up, 8 in 2024-04-26T05:01:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:34 smithi005 ceph-mon[91801]: pgmap v976: 105 pgs: 105 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 55 MiB/s wr, 286 op/s 2024-04-26T05:01:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:34 smithi005 ceph-mon[91801]: osdmap e861: 8 total, 8 up, 8 in 2024-04-26T05:01:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:34 smithi111 ceph-mon[86360]: pgmap v976: 105 pgs: 105 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 55 MiB/s wr, 286 op/s 2024-04-26T05:01:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:34 smithi111 ceph-mon[86360]: osdmap e861: 8 total, 8 up, 8 in 2024-04-26T05:01:35.908 INFO:tasks.workunit.client.0.smithi005.stdout: i=10 exists [4066564~17219964,22470476~4781207,27326766~12581856,44836370~8184528,55397009~17358876,72793401~9073393,84186761~16846589,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,181493205~28657946,214138291~10029115,226471898~6634796,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,287138561~4778574,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] wrote [16659593~1748287,22694932~4556751,64260123~3449728,84186761~1603742,115339083~475404,233204828~7303640,311409106~471389,316749913~2883833,359690727~4958740,383510826~6665543] 2024-04-26T05:01:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:35 smithi005 ceph-mon[91801]: osdmap e862: 8 total, 8 up, 8 in 2024-04-26T05:01:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:35 smithi005 ceph-mon[95809]: osdmap e862: 8 total, 8 up, 8 in 2024-04-26T05:01:36.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:35 smithi111 ceph-mon[86360]: osdmap e862: 8 total, 8 up, 8 in 2024-04-26T05:01:36.925 INFO:tasks.workunit.client.0.smithi005.stdout: i=11 exists [4066564~17219964,22470476~17438146,44836370~8184528,55397009~17358876,72793401~9073393,84186761~16846589,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,181493205~32378221,214138291~10029115,226471898~6634796,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,287138561~4778574,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] wrote [25627621~4241210,34972837~4732990,86520737~1359758,101273361~3260501,141777995~7228088,208442097~5429329,217239812~6185885,248555976~3677622,282785062~2730194] 2024-04-26T05:01:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:36 smithi005 ceph-mon[95809]: pgmap v979: 105 pgs: 105 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 25 MiB/s wr, 270 op/s 2024-04-26T05:01:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:36 smithi005 ceph-mon[95809]: osdmap e863: 8 total, 8 up, 8 in 2024-04-26T05:01:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:36 smithi005 ceph-mon[91801]: pgmap v979: 105 pgs: 105 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 25 MiB/s wr, 270 op/s 2024-04-26T05:01:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:36 smithi005 ceph-mon[91801]: osdmap e863: 8 total, 8 up, 8 in 2024-04-26T05:01:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:36 smithi111 ceph-mon[86360]: pgmap v979: 105 pgs: 105 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 25 MiB/s wr, 270 op/s 2024-04-26T05:01:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:36 smithi111 ceph-mon[86360]: osdmap e863: 8 total, 8 up, 8 in 2024-04-26T05:01:37.935 INFO:tasks.workunit.client.0.smithi005.stdout: i=12 exists [4066564~17219964,22470476~17438146,44836370~8184528,55397009~17358876,72793401~10087831,84186761~16846589,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,181493205~32378221,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,287138561~4991792,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] wrote [76265946~6615286,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,294153965~6748055,326595993~7971793] 2024-04-26T05:01:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:37 smithi005 ceph-mon[91801]: osdmap e864: 8 total, 8 up, 8 in 2024-04-26T05:01:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:37 smithi005 ceph-mon[95809]: osdmap e864: 8 total, 8 up, 8 in 2024-04-26T05:01:38.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:37 smithi111 ceph-mon[86360]: osdmap e864: 8 total, 8 up, 8 in 2024-04-26T05:01:38.949 INFO:tasks.workunit.client.0.smithi005.stdout: i=13 exists [814559~20471969,22470476~17438146,44836370~8184528,55397009~17358876,72793401~28239949,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,181493205~32378221,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,287138561~4991792,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] wrote [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,82879792~5701829,139237830~2273604,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:39.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:38 smithi005 ceph-mon[95809]: pgmap v982: 105 pgs: 105 active+clean; 440 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 23 MiB/s wr, 13 op/s 2024-04-26T05:01:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:38 smithi005 ceph-mon[95809]: osdmap e865: 8 total, 8 up, 8 in 2024-04-26T05:01:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:38 smithi005 ceph-mon[91801]: pgmap v982: 105 pgs: 105 active+clean; 440 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 23 MiB/s wr, 13 op/s 2024-04-26T05:01:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:38 smithi005 ceph-mon[91801]: osdmap e865: 8 total, 8 up, 8 in 2024-04-26T05:01:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:38 smithi111 ceph-mon[86360]: pgmap v982: 105 pgs: 105 active+clean; 440 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 23 MiB/s wr, 13 op/s 2024-04-26T05:01:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:38 smithi111 ceph-mon[86360]: osdmap e865: 8 total, 8 up, 8 in 2024-04-26T05:01:39.935 INFO:tasks.workunit.client.0.smithi005.stdout: i=14 exists [814559~20471969,22470476~17438146,40504854~12516044,55397009~17358876,72793401~28239949,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,181493205~32378221,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,287138561~18954252,308835410~7134981,316749913~17817873,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] wrote [40504854~4916540,73504814~1175543,101089246~7603736,161552797~233185,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~4874710] 2024-04-26T05:01:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:39 smithi005 ceph-mon[91801]: osdmap e866: 8 total, 8 up, 8 in 2024-04-26T05:01:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:39 smithi005 ceph-mon[95809]: osdmap e866: 8 total, 8 up, 8 in 2024-04-26T05:01:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:39 smithi111 ceph-mon[86360]: osdmap e866: 8 total, 8 up, 8 in 2024-04-26T05:01:40.960 INFO:tasks.workunit.client.0.smithi005.stdout: i=15 exists [814559~20471969,22470476~17438146,40504854~12516044,55397009~17358876,72793401~28239949,101089246~7603736,112016011~7352303,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,181493205~32378221,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,287138561~28831830,316749913~17817873,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] wrote [12820204~6774326,52116421~513575,85238503~5614198,127854997~7118382,190002781~7584905,200091224~4513414,299789837~9158360,363806101~2087299,369299132~1019592] 2024-04-26T05:01:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:40 smithi005 ceph-mon[91801]: pgmap v985: 105 pgs: 105 active+clean; 528 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 45 MiB/s wr, 314 op/s 2024-04-26T05:01:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:40 smithi005 ceph-mon[91801]: osdmap e867: 8 total, 8 up, 8 in 2024-04-26T05:01:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:40 smithi005 ceph-mon[95809]: pgmap v985: 105 pgs: 105 active+clean; 528 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 45 MiB/s wr, 314 op/s 2024-04-26T05:01:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:40 smithi005 ceph-mon[95809]: osdmap e867: 8 total, 8 up, 8 in 2024-04-26T05:01:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:40 smithi111 ceph-mon[86360]: pgmap v985: 105 pgs: 105 active+clean; 528 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 45 MiB/s wr, 314 op/s 2024-04-26T05:01:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:40 smithi111 ceph-mon[86360]: osdmap e867: 8 total, 8 up, 8 in 2024-04-26T05:01:41.975 INFO:tasks.workunit.client.0.smithi005.stdout: i=16 exists [814559~20471969,22470476~30550422,55397009~17358876,72793401~28239949,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~32872106,214138291~39463254,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,287138561~28831830,316749913~18342006,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] wrote [24844439~7388720,38082429~7946871,108350706~7835750,180999320~6640455,197058233~8020163,232621452~5631968,268485267~2543234,307050062~5777289,325766628~2902956,332067187~3024732] 2024-04-26T05:01:42.148 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:41 smithi005 ceph-mon[95809]: osdmap e868: 8 total, 8 up, 8 in 2024-04-26T05:01:42.148 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:41 smithi005 ceph-mon[91801]: osdmap e868: 8 total, 8 up, 8 in 2024-04-26T05:01:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:41 smithi111 ceph-mon[86360]: osdmap e868: 8 total, 8 up, 8 in 2024-04-26T05:01:42.926 INFO:tasks.workunit.client.0.smithi005.stdout: i=17 exists [814559~20471969,22470476~50285409,72793401~28239949,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~32872106,214138291~39463254,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,287138561~28831830,316749913~18342006,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] wrote [50191347~5428687,104551092~5709199,126499545~375794,137230096~4951286,206811004~5690052,235266241~1774805,257797223~2913328,297925974~110671,311160244~70212,337362700~1413194] 2024-04-26T05:01:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:42 smithi005 ceph-mon[95809]: pgmap v988: 105 pgs: 105 active+clean; 618 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 195 KiB/s rd, 46 MiB/s wr, 315 op/s 2024-04-26T05:01:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:42 smithi005 ceph-mon[95809]: osdmap e869: 8 total, 8 up, 8 in 2024-04-26T05:01:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:42 smithi005 ceph-mon[91801]: pgmap v988: 105 pgs: 105 active+clean; 618 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 195 KiB/s rd, 46 MiB/s wr, 315 op/s 2024-04-26T05:01:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:42 smithi005 ceph-mon[91801]: osdmap e869: 8 total, 8 up, 8 in 2024-04-26T05:01:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:42 smithi111 ceph-mon[86360]: pgmap v988: 105 pgs: 105 active+clean; 618 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 195 KiB/s rd, 46 MiB/s wr, 315 op/s 2024-04-26T05:01:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:42 smithi111 ceph-mon[86360]: osdmap e869: 8 total, 8 up, 8 in 2024-04-26T05:01:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:01:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:01:43] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.43.0" 2024-04-26T05:01:43.971 INFO:tasks.workunit.client.0.smithi005.stdout: i=18 exists [814559~20471969,22470476~50285409,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~32872106,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,287138561~28831830,316749913~34162978,355398337~3388765,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] wrote [89273018~1622147,98101745~1226828,143863691~8561016,222589835~3430799,270287176~4451997,334915611~7171348,355398337~3388765] 2024-04-26T05:01:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:43 smithi005 ceph-mon[91801]: osdmap e870: 8 total, 8 up, 8 in 2024-04-26T05:01:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:43 smithi005 ceph-mon[95809]: osdmap e870: 8 total, 8 up, 8 in 2024-04-26T05:01:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:43 smithi111 ceph-mon[86360]: osdmap e870: 8 total, 8 up, 8 in 2024-04-26T05:01:44.988 INFO:tasks.workunit.client.0.smithi005.stdout: i=19 exists [814559~20471969,22470476~50285409,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~32872106,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,287138561~28831830,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] wrote [4666460~3132404,25041300~7162615,47571455~1002103,139295633~5658608,147983914~4811081,166860726~2585567,215902769~5180291,311000057~4929630,364904528~8203358] 2024-04-26T05:01:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:44 smithi005 ceph-mon[91801]: pgmap v991: 105 pgs: 105 active+clean; 730 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 52 MiB/s wr, 362 op/s 2024-04-26T05:01:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:44 smithi005 ceph-mon[91801]: osdmap e871: 8 total, 8 up, 8 in 2024-04-26T05:01:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:44 smithi005 ceph-mon[95809]: pgmap v991: 105 pgs: 105 active+clean; 730 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 52 MiB/s wr, 362 op/s 2024-04-26T05:01:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:44 smithi005 ceph-mon[95809]: osdmap e871: 8 total, 8 up, 8 in 2024-04-26T05:01:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:44 smithi111 ceph-mon[86360]: pgmap v991: 105 pgs: 105 active+clean; 730 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 231 KiB/s rd, 52 MiB/s wr, 362 op/s 2024-04-26T05:01:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:44 smithi111 ceph-mon[86360]: osdmap e871: 8 total, 8 up, 8 in 2024-04-26T05:01:45.920 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 1 diff [29081750~7590467,56441850~5272261,84444570~14520168,137672071~4179285,187456785~1896585,192002560~3429953,200091691~1277945,238186251~6498645,363721045~8096572] 2024-04-26T05:01:45.920 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [29081750~7590467,56441850~5272261,84444570~14520168,137672071~4179285,187456785~1896585,192002560~3429953,200091691~1277945,238186251~6498645,363721045~8096572] 2024-04-26T05:01:45.938 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [29081750~7590467,56441850~5272261,83886080~15078658,134217728~7633628,187456785~7975728,197132288~4237348,234881024~9803872,360710144~11107473] 2024-04-26T05:01:45.938 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 2 diff [29081750~7590467,44836370~8184528,56441850~5272261,84444570~14520168,132942611~8908745,157730584~617997,166051702~5657552,187456785~1896585,192002560~3429953,200091691~1277945,238186251~6498645,269611654~850468,300090207~6002606,327281779~6554677,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.938 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [29081750~7590467,44836370~8184528,56441850~5272261,84444570~14520168,132942611~8908745,157730584~617997,166051702~5657552,187456785~1896585,192002560~3429953,200091691~1277945,238186251~6498645,269611654~850468,300090207~6002606,327281779~6554677,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.945 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [29081750~7590467,41943040~11077858,56441850~5272261,83886080~15078658,130023424~11827932,155189248~3159333,163577856~8131398,187456785~7975728,197132288~4237348,234881024~9803872,268435456~2026666,297795584~8297229,327155712~6680744,360710144~11107473,373293056~6425220,394264576~2077530] 2024-04-26T05:01:45.945 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 3 diff [29081750~7590467,44836370~8184528,56441850~5272261,66421625~6334260,77060329~2914962,84444570~14520168,123388044~663063,132942611~8908745,143398206~3113874,157730584~617997,166051702~5657552,187456785~4400963,192002560~3429953,200091691~1277945,238186251~6498645,255108052~876750,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.945 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [29081750~7590467,44836370~8184528,56441850~5272261,66421625~6334260,77060329~2914962,84444570~14520168,123388044~663063,132942611~8908745,143398206~3113874,157730584~617997,166051702~5657552,187456785~4400963,192002560~3429953,200091691~1277945,238186251~6498645,255108052~876750,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.952 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [29081750~7590467,41943040~11077858,56441850~5272261,62914560~9841325,75497472~4477819,83886080~15078658,121634816~2416291,130023424~11827932,142606336~3905744,155189248~3159333,163577856~8131398,187456785~7975728,197132288~4237348,234881024~9803872,251658240~4326562,268435456~2026666,297795584~8297229,306184192~17541479,327155712~6680744,360710144~11107473,373293056~6425220,394264576~2077530] 2024-04-26T05:01:45.952 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 4 diff [29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~7181890,84444570~14520168,123388044~663063,132942611~8908745,143398206~3113874,151760462~7153854,166051702~5657552,187456785~4400963,192002560~9367076,214138291~4515375,226471898~6634796,238186251~8595190,255108052~876750,261080148~7565141,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,346194868~2230218,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.953 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~7181890,84444570~14520168,123388044~663063,132942611~8908745,143398206~3113874,151760462~7153854,166051702~5657552,187456785~4400963,192002560~9367076,214138291~4515375,226471898~6634796,238186251~8595190,255108052~876750,261080148~7565141,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,346194868~2230218,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.960 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [29081750~7590467,41943040~11077858,56441850~23533441,83886080~15078658,121634816~2416291,130023424~11827932,142606336~3905744,150994944~7919372,163577856~8131398,187456785~13912851,213909504~4744162,224167406~8939288,234881024~11900417,251658240~4326562,260046848~10415274,297795584~8297229,306184192~17541479,327155712~6680744,344262007~4163079,360710144~11107473,373293056~6425220,394264576~2077530] 2024-04-26T05:01:45.960 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 5 diff [29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~7181890,81459847~22196,84444570~16588780,123388044~663063,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,187456785~4400963,192002560~9367076,214138291~4515375,226471898~6634796,238186251~8595190,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~6979400,317187470~7629265,327281779~6554677,335978829~661345,346194868~2230218,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.960 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~7181890,81459847~22196,84444570~16588780,123388044~663063,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,187456785~4400963,192002560~9367076,214138291~4515375,226471898~6634796,238186251~8595190,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~6979400,317187470~7629265,327281779~6554677,335978829~661345,346194868~2230218,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.967 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [29081750~7590467,41943040~11077858,56441850~25040193,83886080~17147270,121634816~2416291,130023424~11827932,142606336~3905744,146800640~12113676,163577856~8131398,187456785~13912851,213909504~4744162,224167406~8939288,234881024~11900417,251658240~18803882,297795584~8297229,306184192~18632543,327155712~6680744,335544320~1095854,344262007~4163079,360710144~11107473,373293056~6425220,394264576~2077530] 2024-04-26T05:01:45.969 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 6 diff [29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,187456785~4400963,192002560~10699222,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,342966963~7857771,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.969 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,187456785~4400963,192002560~10699222,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,342966963~7857771,360818268~10999349,375480946~4237330,394539851~1802255] 2024-04-26T05:01:45.974 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [29081750~7590467,41943040~11077858,56441850~25424944,83886080~17147270,109051904~10316410,121634816~2416291,130023424~11827932,142606336~3905744,146800640~12113676,163577856~8131398,184198593~2978290,187456785~15244997,213909504~8857230,224167406~8939288,234881024~35581098,297795584~8297229,306184192~18632543,327155712~6680744,335544320~1095854,342966963~7857771,360710144~11107473,373293056~6425220,394264576~2077530] 2024-04-26T05:01:45.974 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 7 diff [9035947~8048222,29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,187456785~4400963,192002560~10699222,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,394539851~3440498] 2024-04-26T05:01:45.974 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,29081750~7590467,44836370~8184528,56441850~6765436,66421625~6334260,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,187456785~4400963,192002560~10699222,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,394539851~3440498] 2024-04-26T05:01:45.981 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,29081750~7590467,41943040~11077858,56441850~25424944,83886080~17147270,109051904~10316410,121634816~2416291,125829120~16022236,142606336~3905744,146800640~12113676,163577856~8131398,184198593~2978290,187456785~15244997,213909504~8857230,224167406~8939288,234881024~35581098,272629760~3613758,281018368~5084137,297795584~8297229,306184192~18632543,327155712~6680744,335544320~15368571,360710144~11107473,373293056~8173020,394264576~3715773] 2024-04-26T05:01:45.981 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 8 diff [4066564~13017605,22470476~1110648,29081750~10826872,44836370~8184528,56441850~16314035,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5657552,184198593~2978290,187456785~4400963,192002560~14320600,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,394539851~3440498] 2024-04-26T05:01:45.981 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,29081750~10826872,44836370~8184528,56441850~16314035,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5657552,184198593~2978290,187456785~4400963,192002560~14320600,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,394539851~3440498] 2024-04-26T05:01:45.988 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,29081750~10826872,41943040~11077858,56441850~25424944,83886080~17147270,109051904~10316410,121634816~2416291,125829120~16022236,142606336~3905744,146800640~12113676,163577856~8131398,184198593~2978290,187456785~18866375,213909504~8857230,224167406~8939288,234881024~35581098,272629760~3613758,281018368~5084137,297795584~8297229,306184192~18632543,327155712~6680744,335544320~15368571,360710144~11107473,373293056~8173020,394264576~3715773] 2024-04-26T05:01:45.988 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 9 diff [4066564~17219964,22470476~1110648,29081750~10826872,44836370~8184528,56441850~16314035,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~9659816,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,382845155~4560594,394539851~3440498] 2024-04-26T05:01:45.988 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,29081750~10826872,44836370~8184528,56441850~16314035,72793401~9073393,84444570~16588780,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,214138291~8628443,226471898~6634796,238186251~15415294,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~9659816,327281779~6554677,335978829~661345,338235024~12677867,360818268~10999349,375480946~5985130,382845155~4560594,394539851~3440498] 2024-04-26T05:01:45.995 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,29081750~10826872,41943040~11077858,56441850~25424944,83886080~17147270,109051904~10316410,121634816~2416291,125829120~16022236,142606336~3905744,146800640~12113676,163577856~8469813,184198593~2978290,187456785~18866375,213909504~8857230,224167406~8939288,234881024~35581098,272629760~3613758,281018368~5084137,297795584~8297229,306184192~20663094,327155712~6680744,335544320~15368571,360710144~11107473,373293056~8173020,381681664~5724085,394264576~3715773] 2024-04-26T05:01:45.995 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 10 diff [4066564~17219964,22470476~4781207,29081750~10826872,44836370~8184528,56441850~16314035,72793401~9073393,84186761~16846589,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,214138291~8628443,226471898~6634796,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:45.995 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,29081750~10826872,44836370~8184528,56441850~16314035,72793401~9073393,84186761~16846589,112016011~7352303,123388044~663063,125863971~6236078,132942611~8908745,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,214138291~8628443,226471898~6634796,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.001 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,29081750~10826872,41943040~11077858,56441850~25424944,83886080~17147270,109051904~10316410,121634816~2416291,125829120~16022236,142606336~3905744,146800640~12113676,163577856~8469813,184198593~2978290,187456785~18866375,213909504~8857230,224167406~46294716,272629760~3613758,281018368~5084137,297795584~8297229,306184192~20663094,327155712~6680744,335544320~15368571,356515840~15301777,373293056~8173020,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.001 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 11 diff [4066564~17219964,22470476~17438146,44836370~8184528,56441850~16314035,72793401~9073393,84186761~16846589,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~9287406,226471898~6634796,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.001 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~17438146,44836370~8184528,56441850~16314035,72793401~9073393,84186761~16846589,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~9287406,226471898~6634796,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.007 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~39908622,41943040~11077858,56441850~25424944,83886080~20647782,109051904~10316410,121634816~2416291,125829120~33085196,163577856~8469813,184198593~2978290,187456785~18866375,208442097~5429329,213909504~9516193,224167406~46294716,272629760~3613758,281018368~5084137,297795584~8297229,306184192~20663094,327155712~6680744,335544320~15368571,356515840~15301777,373293056~8173020,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.008 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 12 diff [4066564~17219964,22470476~17438146,44836370~8184528,56441850~16314035,72793401~10087831,84186761~16846589,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.008 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~17438146,44836370~8184528,56441850~16314035,72793401~10087831,84186761~16846589,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~12126890,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.014 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~39908622,41943040~11077858,56441850~26439382,83886080~20647782,109051904~10316410,121634816~2416291,125829120~33085196,163577856~8469813,184198593~2978290,187456785~18866375,208442097~5429329,213909504~56552618,272629760~3613758,281018368~5084137,288524467~3605886,293601280~12491533,306184192~28383594,335544320~15368571,356515840~15301777,373293056~8173020,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.014 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 13 diff [814559~20471969,22470476~17438146,44836370~8184528,56441850~16314035,72793401~28239949,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.014 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~17438146,44836370~8184528,56441850~16314035,72793401~28239949,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.020 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~39908622,41943040~11077858,56441850~48092012,109051904~10316410,121634816~2416291,125829120~33085196,163577856~8469813,184198593~2978290,187456785~18866375,208442097~5429329,213909504~56552618,272629760~3613758,281018368~5084137,288524467~3605886,293601280~12491533,306184192~28383594,335544320~15368571,356515840~24950236,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.020 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 14 diff [814559~20471969,22470476~17438146,40504854~12516044,56441850~16314035,72793401~28239949,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~17568346,308835410~7134981,316749913~17817873,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.020 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~17438146,40504854~12516044,56441850~16314035,72793401~28239949,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,132942611~18495679,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~17568346,308835410~7134981,316749913~17817873,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.027 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~53020898,56441850~52251132,109051904~10316410,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,184198593~2978290,187456785~18866375,208442097~5429329,213909504~56552618,272629760~3613758,281018368~5084137,288524467~17568346,306184192~28383594,335544320~15368571,356515840~24950236,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.027 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 15 diff [814559~20471969,22470476~17438146,40504854~12516044,56441850~16314035,72793401~28239949,101089246~7603736,112016011~7352303,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~17817873,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.027 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~17438146,40504854~12516044,56441850~16314035,72793401~28239949,101089246~7603736,112016011~7352303,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,187456785~18866375,208442097~5429329,214138291~18968403,233204828~20396717,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~17817873,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.033 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~53020898,56441850~52251132,109051904~10316410,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,184198593~2978290,187456785~18866375,208442097~5429329,213909504~56552618,272629760~3613758,281018368~5084137,288524467~46043319,335544320~15368571,356515840~24950236,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.034 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 16 diff [814559~20471969,22470476~30550422,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,208442097~5429329,214138291~39463254,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.034 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~30550422,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,208442097~5429329,214138291~39463254,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.040 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~53020898,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,208442097~5429329,213909504~57118997,272629760~3613758,281018368~5084137,288524467~46567452,335544320~15368571,356515840~24950236,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.040 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 17 diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.040 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.046 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~57118997,272629760~3613758,281018368~5084137,288524467~46567452,335544320~15368571,356515840~24950236,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.047 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 18 diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.047 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~12126890,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.053 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.053 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 19 diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.053 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.059 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.059 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 2 diff [44836370~8184528,132942611~6895907,157730584~617997,166051702~5657552,269611654~850468,300090207~6002606,327281779~6554677,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.059 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [44836370~8184528,132942611~6895907,157730584~617997,166051702~5657552,269611654~850468,300090207~6002606,327281779~6554677,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.064 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [41943040~11077858,130023424~9815094,155189248~3159333,163577856~8131398,268435456~2026666,297795584~8297229,327155712~6680744,360818268~6433092,373293056~6425220,394264576~2077530] 2024-04-26T05:01:46.065 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 3 diff [44836370~8184528,66421625~6334260,77060329~2914962,123388044~663063,132942611~6895907,143398206~3113874,157730584~617997,166051702~5657552,188506823~3350925,255108052~876750,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.065 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [44836370~8184528,66421625~6334260,77060329~2914962,123388044~663063,132942611~6895907,143398206~3113874,157730584~617997,166051702~5657552,188506823~3350925,255108052~876750,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.070 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [41943040~11077858,62914560~9841325,75497472~4477819,121634816~2416291,130023424~9815094,142606336~3905744,155189248~3159333,163577856~8131398,188506823~3350925,251658240~4326562,268435456~2026666,297795584~8297229,306184192~17541479,327155712~6680744,360818268~6433092,373293056~6425220,394264576~2077530] 2024-04-26T05:01:46.071 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 4 diff [44836370~8184528,61115753~2091533,66421625~6334260,72793401~7181890,123388044~663063,132942611~6895907,143398206~3113874,151760462~7153854,166051702~5657552,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,261080148~7565141,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,346194868~2230218,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.071 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [44836370~8184528,61115753~2091533,66421625~6334260,72793401~7181890,123388044~663063,132942611~6895907,143398206~3113874,151760462~7153854,166051702~5657552,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,261080148~7565141,269611654~850468,300090207~6002606,308835410~6979400,317187470~6538201,327281779~6554677,346194868~2230218,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.077 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [41943040~11077858,61115753~18859538,121634816~2416291,130023424~9815094,142606336~3905744,150994944~7919372,163577856~8131398,188506823~3350925,193840629~6595274,213909504~4744162,224167406~8939288,244060445~2720996,251658240~4326562,260046848~10415274,297795584~8297229,306184192~17541479,327155712~6680744,344262007~4163079,360818268~6433092,373293056~6425220,394264576~2077530] 2024-04-26T05:01:46.077 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 5 diff [44836370~8184528,56614553~6592733,66421625~6334260,72793401~7181890,81459847~22196,93866620~7166730,123388044~663063,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~6979400,317187470~7629265,327281779~6554677,335978829~661345,346194868~2230218,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.077 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [44836370~8184528,56614553~6592733,66421625~6334260,72793401~7181890,81459847~22196,93866620~7166730,123388044~663063,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~6979400,317187470~7629265,327281779~6554677,335978829~661345,346194868~2230218,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.083 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [41943040~11077858,56614553~24867490,93866620~7166730,121634816~2416291,130023424~9815094,142606336~3905744,146800640~12113676,163577856~8131398,188506823~3350925,193840629~6595274,213909504~4744162,224167406~8939288,244060445~2720996,251658240~18803882,297795584~8297229,306184192~18632543,327155712~6680744,335544320~1095854,344262007~4163079,360818268~6433092,373293056~6425220,394264576~2077530] 2024-04-26T05:01:46.083 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 6 diff [44836370~8184528,56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,342966963~7857771,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.083 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [44836370~8184528,56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,342966963~7857771,360818268~6433092,375480946~4237330,394539851~1802255] 2024-04-26T05:01:46.089 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [41943040~11077858,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,130023424~9815094,142606336~3905744,146800640~12113676,163577856~8131398,184198593~2978290,188506823~3350925,193840629~8861153,213909504~8857230,224167406~8939288,244060445~26401677,297795584~8297229,306184192~18632543,327155712~6680744,335544320~1095854,342966963~7857771,360818268~6433092,373293056~6425220,394264576~2077530] 2024-04-26T05:01:46.089 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 7 diff [9035947~8048222,44836370~8184528,56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~6433092,375480946~5985130,394539851~3440498] 2024-04-26T05:01:46.089 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,44836370~8184528,56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,166051702~5657552,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~6433092,375480946~5985130,394539851~3440498] 2024-04-26T05:01:46.095 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,41943040~11077858,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,125829120~14009398,142606336~3905744,146800640~12113676,163577856~8131398,184198593~2978290,188506823~3350925,193840629~8861153,213909504~8857230,224167406~8939288,244060445~26401677,272629760~3613758,281018368~5084137,297795584~8297229,306184192~18632543,327155712~6680744,335544320~15368571,360818268~6433092,373293056~8173020,394264576~3715773] 2024-04-26T05:01:46.095 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 8 diff [4066564~13017605,22470476~1110648,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5657552,184198593~2978290,188506823~3350925,193840629~12482531,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~6433092,375480946~5985130,394539851~3440498] 2024-04-26T05:01:46.095 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5657552,184198593~2978290,188506823~3350925,193840629~12482531,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~7629265,327281779~6554677,335978829~661345,338235024~12677867,360818268~6433092,375480946~5985130,394539851~3440498] 2024-04-26T05:01:46.101 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,32116244~7792378,41943040~11077858,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,125829120~14009398,142606336~3905744,146800640~12113676,163577856~8131398,184198593~2978290,188506823~3350925,193840629~12482531,213909504~8857230,224167406~8939288,244060445~26401677,272629760~3613758,281018368~5084137,297795584~8297229,306184192~18632543,327155712~6680744,335544320~15368571,360818268~6433092,373293056~8173020,394264576~3715773] 2024-04-26T05:01:46.101 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~9659816,327281779~6554677,335978829~661345,338235024~12677867,360818268~6433092,375480946~5985130,382845155~4560594,394539851~3440498] 2024-04-26T05:01:46.101 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,317187470~9659816,327281779~6554677,335978829~661345,338235024~12677867,360818268~6433092,375480946~5985130,382845155~4560594,394539851~3440498] 2024-04-26T05:01:46.107 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,41943040~11077858,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,125829120~14009398,142606336~3905744,146800640~12113676,163577856~8469813,184198593~2978290,188506823~17816337,213909504~8857230,224167406~8939288,244060445~26401677,272629760~3613758,281018368~5084137,297795584~8297229,306184192~20663094,327155712~6680744,335544320~15368571,360818268~6433092,373293056~8173020,381681664~5724085,394264576~3715773] 2024-04-26T05:01:46.107 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,84186761~1603742,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~7560633,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.107 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,84186761~1603742,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,283756946~2345559,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~7560633,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.113 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,41943040~11077858,56614553~25252241,84186761~1603742,93866620~7166730,109051904~10316410,121634816~2416291,125829120~14009398,142606336~3905744,146800640~12113676,163577856~8469813,184198593~2978290,188506823~17816337,213909504~8857230,224167406~16341062,244060445~26401677,272629760~3613758,281018368~5084137,297795584~8297229,306184192~20663094,327155712~6680744,335544320~15368571,356515840~10735520,373293056~8173020,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.113 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,141777995~9660295,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,208442097~5429329,214138291~9287406,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~7560633,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.114 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,44836370~8184528,56614553~16141332,72793401~9073393,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,141777995~9660295,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,208442097~5429329,214138291~9287406,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,300090207~6002606,308835410~7134981,316749913~10097373,327281779~6554677,335978829~661345,338235024~12677867,359690727~7560633,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.119 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,41943040~11077858,56614553~25252241,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,121634816~2416291,125829120~14009398,141777995~17136321,163577856~8469813,184198593~2978290,188506823~17816337,208442097~5429329,213909504~9516193,224167406~16341062,244060445~26401677,272629760~3613758,281018368~5084137,297795584~8297229,306184192~20663094,327155712~6680744,335544320~15368571,356515840~10735520,373293056~8173020,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.120 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,44836370~8184528,56614553~16141332,72793401~10087831,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,141777995~9660295,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~7560633,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.120 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,44836370~8184528,56614553~16141332,72793401~10087831,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~6895907,141777995~9660295,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~7560633,375480946~5985130,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.125 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,41943040~11077858,56614553~26266679,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,121634816~2416291,125829120~14009398,141777995~17136321,163577856~8469813,184198593~2978290,188506823~17816337,208442097~5429329,213909504~26598964,244060445~26401677,272629760~3613758,281018368~5084137,288524467~3605886,293601280~12491533,306184192~28383594,335544320~15368571,356515840~10735520,373293056~8173020,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.125 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,44836370~8184528,56614553~16141332,72793401~15788220,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~8568823,141777995~9660295,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~9481777,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.125 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,44836370~8184528,56614553~16141332,72793401~15788220,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,132942611~8568823,141777995~9660295,151760462~7153854,164569697~386756,166051702~5995967,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273362189~2881329,282785062~3317443,288524467~3605886,294153965~11938848,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~9481777,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.131 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,41943040~11077858,56614553~31967068,93866620~10667242,109051904~10316410,121634816~2416291,125829120~15682314,141777995~17136321,163577856~8469813,184198593~2978290,188506823~17816337,208442097~5429329,213909504~26598964,244060445~26401677,272629760~3613758,281018368~5084137,288524467~3605886,293601280~12491533,306184192~28383594,335544320~15368571,356515840~12656664,371817617~9648459,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.132 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~12516044,56614553~16141332,72793401~15788220,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,132942611~8568823,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~17568346,308835410~7134981,316749913~17817873,335899505~15013386,359690727~9481777,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.132 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~12516044,56614553~16141332,72793401~15788220,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,132942611~8568823,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~17568346,308835410~7134981,316749913~17817873,335899505~15013386,359690727~9481777,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.137 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~20904654,56614553~31967068,93866620~14826362,109051904~10316410,121634816~2416291,125829120~15682314,141777995~17136321,159383552~2402430,163577856~8469813,184198593~2978290,188005676~18317484,208442097~5429329,213909504~26598964,244060445~26401677,272629760~3613758,281018368~5084137,288524467~17568346,306184192~28383594,335544320~15368571,356515840~12656664,371817617~9648459,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.137 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~12516044,56614553~16141332,72793401~18059300,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~15647463,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~17817873,335899505~15013386,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.138 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~12516044,56614553~16141332,72793401~18059300,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~15647463,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,269611654~850468,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~17817873,335899505~15013386,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.144 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~20904654,56614553~34238148,93866620~14826362,109051904~10316410,121634816~2416291,125829120~15682314,141777995~17136321,159383552~2402430,163577856~8469813,184198593~2978290,188005676~18317484,208442097~5429329,213909504~26598964,244060445~26401677,272629760~3613758,281018368~5084137,288524467~46043319,335544320~15368571,356515840~12656664,369299132~1019592,371817617~9648459,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.144 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 16 diff [814559~20471969,22470476~30550422,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~15647463,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,208442097~5429329,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.144 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~30550422,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~15647463,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,208442097~5429329,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.150 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~53020898,56614553~34238148,93866620~25501694,121634816~2416291,125829120~15682314,141777995~17136321,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,208442097~5429329,213909504~26598964,244060445~26968056,272629760~3613758,281018368~5084137,288524467~46567452,335544320~15368571,356515840~12656664,369299132~1019592,371817617~9648459,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.150 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 17 diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.150 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~25574319,151760462~7153854,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.156 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34238148,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~26968056,272629760~3613758,281018368~5084137,288524467~46567452,335544320~15368571,356515840~12656664,369299132~1019592,371817617~9648459,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.156 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 18 diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.156 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~9481777,369299132~1019592,372335201~9130875,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.162 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~16850968,369299132~1019592,371817617~9648459,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.162 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 19 diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.162 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:46.168 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:46.168 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 3 diff [66421625~6334260,77060329~2914962,123388044~663063,143398206~3113874,188506823~3350925,255108052~876750,303888503~1788567,308835410~6979400,317187470~6538201] 2024-04-26T05:01:46.168 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [66421625~6334260,77060329~2914962,123388044~663063,143398206~3113874,188506823~3350925,255108052~876750,303888503~1788567,308835410~6979400,317187470~6538201] 2024-04-26T05:01:46.174 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [62914560~9841325,75497472~4477819,121634816~2416291,142606336~3905744,188506823~3350925,251658240~4326562,303888503~1788567,306184192~17541479] 2024-04-26T05:01:46.175 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 4 diff [61115753~2091533,66421625~6334260,72793401~7181890,123388044~663063,143398206~3113874,151760462~7153854,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,261080148~7565141,303888503~1788567,308835410~6979400,317187470~6538201,346194868~2230218,364526075~58152] 2024-04-26T05:01:46.175 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [61115753~2091533,66421625~6334260,72793401~7181890,123388044~663063,143398206~3113874,151760462~7153854,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,261080148~7565141,303888503~1788567,308835410~6979400,317187470~6538201,346194868~2230218,364526075~58152] 2024-04-26T05:01:46.181 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [61115753~18859538,121634816~2416291,142606336~3905744,150994944~7919372,188506823~3350925,193840629~6595274,213909504~4744162,224167406~8939288,244060445~2720996,251658240~4326562,260046848~8598441,303888503~1788567,306184192~17541479,344262007~4163079,364526075~58152] 2024-04-26T05:01:46.181 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 5 diff [56614553~6592733,66421625~6334260,72793401~7181890,81459847~22196,93866620~7166730,123388044~663063,143398206~3113874,147744643~3693647,151760462~7153854,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,256469328~12175961,303888503~1788567,308835410~6979400,317187470~7629265,335978829~661345,346194868~2230218,364459877~1335229] 2024-04-26T05:01:46.181 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [56614553~6592733,66421625~6334260,72793401~7181890,81459847~22196,93866620~7166730,123388044~663063,143398206~3113874,147744643~3693647,151760462~7153854,188506823~3350925,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,255108052~876750,256469328~12175961,303888503~1788567,308835410~6979400,317187470~7629265,335978829~661345,346194868~2230218,364459877~1335229] 2024-04-26T05:01:46.187 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [56614553~24867490,93866620~7166730,121634816~2416291,142606336~3905744,146800640~12113676,188506823~3350925,193840629~6595274,213909504~4744162,224167406~8939288,244060445~2720996,251658240~16987049,303888503~1788567,306184192~18632543,335544320~1095854,344262007~4163079,364459877~1335229] 2024-04-26T05:01:46.187 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 6 diff [56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,143398206~3113874,147744643~3693647,151760462~7153854,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,303888503~1788567,308835410~7134981,317187470~7629265,335978829~661345,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.187 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,143398206~3113874,147744643~3693647,151760462~7153854,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,303888503~1788567,308835410~7134981,317187470~7629265,335978829~661345,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.193 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,142606336~3905744,146800640~12113676,184198593~2978290,188506823~3350925,193840629~8861153,213909504~8857230,224167406~8939288,244060445~24584844,303888503~1788567,306184192~18632543,335544320~1095854,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.193 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 7 diff [9035947~8048222,56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,317187470~7629265,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.193 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,56614553~6592733,66421625~6334260,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,184198593~2978290,188506823~3350925,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,317187470~7629265,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.199 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,125829120~6270929,142606336~3905744,146800640~12113676,184198593~2978290,188506823~3350925,193840629~8861153,213909504~8857230,224167406~8939288,244060445~24584844,272629760~3613758,281018368~5084137,303888503~1788567,306184192~18632543,335544320~15368571,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.199 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 8 diff [4066564~13017605,22470476~1110648,32116244~7792378,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,184198593~2978290,188506823~3350925,193840629~12482531,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,317187470~7629265,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.199 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,32116244~7792378,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,184198593~2978290,188506823~3350925,193840629~12482531,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,317187470~7629265,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.204 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,32116244~7792378,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,125829120~6270929,142606336~3905744,146800640~12113676,164569697~386756,184198593~2978290,188506823~3350925,193840629~12482531,213909504~8857230,224167406~8939288,244060445~24584844,272629760~3613758,281018368~5084137,303888503~1788567,306184192~18632543,335544320~15368571,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.205 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,317187470~9659816,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.205 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,56614553~16141332,72793401~9073393,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,317187470~9659816,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.210 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,56614553~25252241,93866620~7166730,109051904~10316410,121634816~2416291,125829120~6270929,142606336~3905744,146800640~12113676,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,213909504~8857230,224167406~8939288,244060445~24584844,272629760~3613758,281018368~5084137,303888503~1788567,306184192~20663094,335544320~15368571,364459877~1335229,378654936~2811140,381681664~5724085,396077477~1902872] 2024-04-26T05:01:46.210 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,56614553~16141332,72793401~9073393,84186761~1603742,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,316749913~10097373,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.210 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,56614553~16141332,72793401~9073393,84186761~1603742,93866620~7166730,112016011~7352303,123388044~663063,125863971~6236078,143398206~3113874,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,214138291~8628443,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,283756946~2345559,303888503~1788567,308835410~7134981,316749913~10097373,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.216 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,56614553~25252241,84186761~1603742,93866620~7166730,109051904~10316410,121634816~2416291,125829120~6270929,142606336~3905744,146800640~12113676,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,213909504~8857230,224167406~16341062,244060445~24584844,272629760~3613758,281018368~5084137,303888503~1788567,306184192~20663094,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.217 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~16141332,72793401~9073393,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,214138291~9287406,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,282785062~3317443,303888503~1788567,308835410~7134981,316749913~10097373,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.217 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~16141332,72793401~9073393,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,214138291~9287406,226471898~6634796,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,282785062~3317443,303888503~1788567,308835410~7134981,316749913~10097373,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.223 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~25252241,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,121634816~2416291,125829120~6270929,141777995~17136321,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,213909504~9516193,224167406~16341062,244060445~24584844,272629760~3613758,281018368~5084137,303888503~1788567,306184192~20663094,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.223 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~16141332,72793401~10087831,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,303888503~1788567,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.223 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~16141332,72793401~10087831,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,303888503~1788567,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.229 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~26266679,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,121634816~2416291,125829120~6270929,141777995~17136321,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,213909504~26598964,244060445~24584844,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,303888503~1788567,306184192~28383594,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.229 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,56614553~16141332,72793401~15788220,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,303888503~1788567,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.229 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,56614553~16141332,72793401~15788220,93866620~7166730,101273361~3260501,112016011~7352303,123388044~663063,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,303888503~1788567,308835410~7134981,316749913~17817873,335978829~661345,338235024~12677867,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:45 smithi005 ceph-mon[91801]: osdmap e872: 8 total, 8 up, 8 in 2024-04-26T05:01:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:45 smithi005 ceph-mon[95809]: osdmap e872: 8 total, 8 up, 8 in 2024-04-26T05:01:46.235 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~31967068,93866620~10667242,109051904~10316410,121634816~2416291,125829120~6270929,139237830~2273604,141777995~17136321,164569697~386756,166755333~5292336,184198593~2978290,188506823~17816337,208442097~5429329,213909504~26598964,244060445~24584844,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,303888503~1788567,306184192~28383594,335544320~15368571,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.235 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56614553~16141332,72793401~15788220,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,303888503~1788567,308835410~7134981,316749913~17817873,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.235 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56614553~16141332,72793401~15788220,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,303888503~1788567,308835410~7134981,316749913~17817873,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.241 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,56614553~31967068,93866620~14826362,109051904~10316410,121634816~2416291,125829120~6270929,139237830~2273604,141777995~17136321,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,213909504~26598964,244060445~24584844,272629760~3613758,281018368~5084137,288524467~12377553,303888503~1788567,306184192~28383594,335544320~15368571,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.241 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56614553~16141332,72793401~18059300,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~17817873,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.241 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56614553~16141332,72793401~18059300,93866620~7166730,101089246~7603736,112016011~7352303,123388044~663063,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,255108052~876750,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~17817873,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.247 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,52116421~513575,56614553~34238148,93866620~14826362,109051904~10316410,121634816~2416291,125829120~9144259,139237830~2273604,141777995~17136321,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,213909504~26598964,244060445~24584844,272629760~3613758,281018368~5084137,288524467~46043319,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.247 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 16 diff [814559~20471969,22470476~23558824,52116421~513575,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.247 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,52116421~513575,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.253 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,52116421~513575,56614553~34238148,93866620~25501694,121634816~2416291,125829120~9144259,139237830~2273604,141777995~17136321,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,213909504~26598964,244060445~26968056,272629760~3613758,281018368~5084137,288524467~46567452,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.253 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 17 diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~14208194,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.253 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~16141332,72793401~18059300,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~14208194,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~27445924,316749913~18342006,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.259 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56614553~34238148,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~26968056,272629760~3613758,281018368~5084137,288524467~46567452,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.259 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 18 diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.259 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.265 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~16850968,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.265 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 19 diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.265 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:45 smithi111 ceph-mon[86360]: osdmap e872: 8 total, 8 up, 8 in 2024-04-26T05:01:46.271 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.271 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 4 diff [61115753~2091533,72793401~4677649,151760462~7153854,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,261080148~7565141,346194868~2230218,364526075~58152] 2024-04-26T05:01:46.271 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [61115753~2091533,72793401~4677649,151760462~7153854,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,261080148~7565141,346194868~2230218,364526075~58152] 2024-04-26T05:01:46.277 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [61115753~2091533,72755885~4715165,150994944~7919372,193840629~6595274,213909504~4744162,224167406~8939288,244060445~2720996,260046848~8598441,344262007~4163079,364526075~58152] 2024-04-26T05:01:46.277 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 5 diff [56614553~6592733,72793401~4677649,81459847~22196,93866620~7166730,147744643~3693647,151760462~7153854,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,256469328~12175961,320955023~3861712,335978829~661345,346194868~2230218,364459877~1335229] 2024-04-26T05:01:46.277 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [56614553~6592733,72793401~4677649,81459847~22196,93866620~7166730,147744643~3693647,151760462~7153854,193840629~6595274,214138291~4515375,226471898~6634796,244060445~2720996,256469328~12175961,320955023~3861712,335978829~661345,346194868~2230218,364459877~1335229] 2024-04-26T05:01:46.282 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [56614553~6592733,72755885~4715165,79975291~1506752,93866620~7166730,146800640~12113676,193840629~6595274,213909504~4744162,224167406~8939288,244060445~2720996,255984802~12660487,320955023~3861712,335544320~1095854,344262007~4163079,364459877~1335229] 2024-04-26T05:01:46.282 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 6 diff [56614553~6592733,72793401~4677649,78964078~2902716,93866620~7166730,112016011~7352303,147744643~3693647,151760462~7153854,184198593~2978290,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,314923259~1047132,320955023~3861712,335978829~661345,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.282 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [56614553~6592733,72793401~4677649,78964078~2902716,93866620~7166730,112016011~7352303,147744643~3693647,151760462~7153854,184198593~2978290,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,314923259~1047132,320955023~3861712,335978829~661345,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.288 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [56614553~6592733,72755885~4715165,78964078~2902716,93866620~7166730,109051904~10316410,146800640~12113676,184198593~2978290,193840629~8861153,213909504~8857230,224167406~8939288,244060445~9541100,255984802~12660487,314923259~1047132,320955023~3861712,335544320~1095854,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.288 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 7 diff [9035947~8048222,56614553~6592733,72793401~4677649,78964078~2902716,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,184198593~2978290,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.288 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,56614553~6592733,72793401~4677649,78964078~2902716,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,184198593~2978290,193840629~8861153,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.294 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,56614553~6592733,72755885~4715165,78964078~2902716,93866620~7166730,109051904~10316410,125829120~6270929,146800640~12113676,184198593~2978290,193840629~8861153,213909504~8857230,224167406~8939288,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,320955023~3861712,335544320~15368571,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.294 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 8 diff [4066564~13017605,22470476~1110648,32116244~7792378,56614553~13663295,72793401~9073393,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,164569697~386756,184198593~2978290,193840629~12482531,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.295 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,32116244~7792378,56614553~13663295,72793401~9073393,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,164569697~386756,184198593~2978290,193840629~12482531,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.300 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,32116244~7792378,56614553~13663295,72755885~9110909,93866620~7166730,109051904~10316410,125829120~6270929,146800640~12113676,164569697~386756,184198593~2978290,193840629~12482531,213909504~8857230,224167406~8939288,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,320955023~3861712,335544320~15368571,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.300 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,56614553~13663295,72793401~9073393,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~5892263,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.300 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,56614553~13663295,72793401~9073393,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,214138291~8628443,226471898~6634796,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~5892263,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.306 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,56614553~13663295,72755885~9110909,93866620~7166730,109051904~10316410,125829120~6270929,146800640~12113676,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,213909504~8857230,224167406~8939288,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,320955023~5892263,335544320~15368571,364459877~1335229,378654936~2811140,381681664~5724085,396077477~1902872] 2024-04-26T05:01:46.306 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,56614553~13663295,72793401~9073393,84186761~1603742,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,214138291~8628443,226471898~6634796,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.306 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,56614553~13663295,72793401~9073393,84186761~1603742,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,214138291~8628443,226471898~6634796,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,283756946~2345559,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.312 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,56614553~13663295,72755885~9110909,84186761~1603742,93866620~7166730,109051904~10316410,125829120~6270929,146800640~12113676,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,213909504~8857230,224167406~16341062,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.312 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,72793401~9073393,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,214138291~9287406,226471898~6634796,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,282785062~3317443,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.313 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,72793401~9073393,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,214138291~9287406,226471898~6634796,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,282785062~3317443,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.318 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~13663295,72755885~9110909,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,125829120~6270929,141777995~17136321,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,213909504~9516193,224167406~16341062,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.318 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,72793401~10087831,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.318 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,72793401~10087831,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.324 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~13663295,72755885~10125347,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,125829120~6270929,141777995~17136321,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,213909504~26598964,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.324 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,56614553~13663295,72793401~15788220,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.324 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,56614553~13663295,72793401~15788220,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.330 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~13663295,72755885~15825736,93866620~10667242,109051904~10316410,125829120~6270929,139237830~2273604,141777995~17136321,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,213909504~26598964,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.330 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56614553~13663295,72793401~15788220,93866620~7166730,101089246~7603736,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.330 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56614553~13663295,72793401~15788220,93866620~7166730,101089246~7603736,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.336 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,56614553~13663295,72755885~15825736,93866620~14826362,109051904~10316410,125829120~6270929,139237830~2273604,141777995~17136321,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,213909504~26598964,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.337 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56614553~13663295,72793401~18059300,93866620~7166730,101089246~7603736,112016011~7352303,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.337 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56614553~13663295,72793401~18059300,93866620~7166730,101089246~7603736,112016011~7352303,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,214138291~18968403,233204828~7303640,244060445~9541100,256469328~12175961,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.342 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,52116421~513575,56614553~13663295,72755885~18096816,93866620~14826362,109051904~10316410,125829120~9144259,139237830~2273604,141777995~17136321,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,213909504~26598964,244060445~9541100,255984802~12660487,272629760~3613758,281018368~5084137,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.343 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 16 diff [814559~20471969,22470476~23558824,52116421~513575,56614553~13663295,72793401~18059300,93866620~7166730,101089246~18279068,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,214138291~26370177,244060445~9541100,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.343 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,52116421~513575,56614553~13663295,72793401~18059300,93866620~7166730,101089246~18279068,125863971~9109408,139237830~2273604,141777995~9660295,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,214138291~26370177,244060445~9541100,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.349 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,52116421~513575,56614553~13663295,72755885~18096816,93866620~25501694,125829120~9144259,139237830~2273604,141777995~17136321,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,213909504~26598964,244060445~9541100,255984802~15043699,272629760~3613758,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.349 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 17 diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,72793401~18059300,93866620~7166730,101089246~18279068,125863971~9109408,137230096~14208194,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.349 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,72793401~18059300,93866620~7166730,101089246~18279068,125863971~9109408,137230096~14208194,151760462~7153854,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~14559173,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.355 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56614553~13663295,72755885~18096816,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~15043699,272629760~3613758,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.355 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 18 diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~29957868,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.355 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~29957868,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.361 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,320955023~29957868,352321536~16850968,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.361 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 19 diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.361 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.367 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.368 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 5 diff [56614553~4963103,81459847~22196,93866620~7166730,147744643~3693647,197328753~1769922,256469328~6792010,320955023~3861712,335978829~661345,364459877~1335229] 2024-04-26T05:01:46.368 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [56614553~4963103,81459847~22196,93866620~7166730,147744643~3693647,197328753~1769922,256469328~6792010,320955023~3861712,335978829~661345,364459877~1335229] 2024-04-26T05:01:46.374 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [56614553~4963103,79975291~1506752,93866620~7166730,146800640~4637650,197328753~1769922,255984802~7276536,320955023~3861712,335544320~1095854,364459877~1335229] 2024-04-26T05:01:46.374 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 6 diff [56614553~4963103,78964078~2902716,93866620~7166730,112016011~7352303,147744643~3693647,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,256469328~6792010,314923259~1047132,320955023~3861712,335978829~661345,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.374 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [56614553~4963103,78964078~2902716,93866620~7166730,112016011~7352303,147744643~3693647,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,256469328~6792010,314923259~1047132,320955023~3861712,335978829~661345,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.381 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [56614553~4963103,78964078~2902716,93866620~7166730,109051904~10316410,146800640~4637650,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,255984802~7276536,314923259~1047132,320955023~3861712,335544320~1095854,342966963~7857771,364459877~1335229] 2024-04-26T05:01:46.381 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 7 diff [9035947~8048222,56614553~4963103,78964078~2902716,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.381 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,56614553~4963103,78964078~2902716,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.386 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,56614553~4963103,78964078~2902716,93866620~7166730,109051904~10316410,125829120~6270929,146800640~4637650,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,255984802~7276536,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,320955023~3861712,335544320~15368571,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.386 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 8 diff [4066564~13017605,22470476~1110648,32116244~7792378,56614553~13663295,77367094~4499700,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,164569697~386756,184198593~2978290,196098904~10224256,217769060~4997674,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.386 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,32116244~7792378,56614553~13663295,77367094~4499700,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,164569697~386756,184198593~2978290,196098904~10224256,217769060~4997674,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~3861712,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.391 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,32116244~7792378,56614553~13663295,77367094~4499700,93866620~7166730,109051904~10316410,125829120~6270929,146800640~4637650,152168489~2837949,164569697~386756,184198593~2978290,196098904~10224256,217769060~4997674,246409527~7192018,255984802~7276536,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,320955023~3861712,335544320~15368571,364459877~1335229,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.391 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,56614553~13663295,77367094~4499700,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~5892263,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.392 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,56614553~13663295,77367094~4499700,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,320955023~5892263,335978829~661345,338235024~12677867,364459877~1335229,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.397 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,56614553~13663295,77367094~4499700,93866620~7166730,109051904~10316410,125829120~6270929,146800640~4637650,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,246409527~7192018,255984802~7276536,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,320955023~5892263,335544320~15368571,364459877~1335229,378654936~2811140,381681664~5724085,396077477~1902872] 2024-04-26T05:01:46.398 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,56614553~13663295,77367094~4499700,84186761~1603742,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,233204828~7303640,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.398 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,56614553~13663295,77367094~4499700,84186761~1603742,93866620~7166730,112016011~7352303,125863971~6236078,147744643~3693647,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,233204828~7303640,246409527~7192018,256469328~6792010,273362189~2881329,283756946~2345559,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.404 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,56614553~13663295,77367094~4499700,84186761~1603742,93866620~7166730,109051904~10316410,125829120~6270929,146800640~4637650,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,233106694~7401774,246409527~7192018,255984802~7276536,272629760~3613758,281018368~5084137,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.404 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,77367094~4499700,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~6185885,233204828~7303640,246409527~7192018,256469328~6792010,273362189~2881329,282785062~3317443,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.404 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,77367094~4499700,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~6185885,233204828~7303640,246409527~7192018,256469328~6792010,273362189~2881329,282785062~3317443,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.409 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~13663295,77367094~4499700,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,125829120~6270929,141777995~9660295,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~6185885,233106694~7401774,246409527~7192018,255984802~7276536,272629760~3613758,281018368~5084137,311409106~471389,314923259~1047132,316749913~3660721,320955023~5892263,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.410 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,76265946~6615286,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.410 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,56614553~13663295,76265946~6615286,84186761~1603742,86520737~1359758,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,141777995~9660295,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~6104379,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.415 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~13663295,76265946~6615286,84186761~1603742,86520737~1359758,93866620~10667242,109051904~10316410,125829120~6270929,141777995~9660295,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,255984802~7276536,264379302~700168,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~9279266,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.416 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,56614553~13663295,76265946~12315675,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.416 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,56614553~13663295,76265946~12315675,93866620~7166730,101273361~3260501,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335978829~661345,338235024~12677867,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.422 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56614553~13663295,76265946~12315675,93866620~10667242,109051904~10316410,125829120~6270929,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,255984802~7276536,264379302~700168,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.422 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56614553~13663295,73504814~1175543,76265946~12315675,93866620~7166730,101089246~7603736,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.422 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56614553~13663295,73504814~1175543,76265946~12315675,93866620~7166730,101089246~7603736,112016011~7352303,125863971~6236078,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.427 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,56614553~13663295,73504814~1175543,76265946~12315675,93866620~14826362,109051904~10316410,125829120~6270929,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,255984802~7276536,264379302~700168,272629760~3613758,281018368~5084137,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.427 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56614553~13663295,73504814~1175543,76265946~14586755,93866620~7166730,101089246~7603736,112016011~7352303,125863971~9109408,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.427 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56614553~13663295,73504814~1175543,76265946~14586755,93866620~7166730,101089246~7603736,112016011~7352303,125863971~9109408,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,256469328~6792010,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.433 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,52116421~513575,56614553~13663295,73504814~1175543,76265946~14586755,93866620~14826362,109051904~10316410,125829120~9144259,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,255984802~7276536,264379302~700168,272629760~3613758,281018368~5084137,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,320955023~13612763,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.434 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 16 diff [814559~20471969,22470476~23558824,52116421~513575,56614553~13663295,73504814~1175543,76265946~14586755,93866620~7166730,101089246~18279068,125863971~9109408,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.434 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,52116421~513575,56614553~13663295,73504814~1175543,76265946~14586755,93866620~7166730,101089246~18279068,125863971~9109408,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.439 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,52116421~513575,56614553~13663295,73504814~1175543,76265946~14586755,93866620~25501694,125829120~9144259,139237830~2273604,141777995~9660295,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~2543234,272629760~3613758,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.439 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 17 diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14586755,93866620~7166730,101089246~18279068,125863971~9109408,137230096~14208194,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.440 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14586755,93866620~7166730,101089246~18279068,125863971~9109408,137230096~14208194,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.445 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14586755,93866620~25501694,125829120~9144259,137230096~14208194,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~2543234,272629760~3613758,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,320955023~14136896,335544320~15368571,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.445 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 18 diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~29957868,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.446 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,320955023~29957868,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.452 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,320955023~29957868,352321536~16850968,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.452 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 19 diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.452 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.458 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.458 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 6 diff [78964078~2902716,112016011~7352303,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,314923259~1047132,342966963~7857771] 2024-04-26T05:01:46.458 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [78964078~2902716,112016011~7352303,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,314923259~1047132,342966963~7857771] 2024-04-26T05:01:46.464 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [78964078~2902716,109051904~10316410,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,314923259~1047132,342966963~7857771] 2024-04-26T05:01:46.464 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 7 diff [9035947~8048222,78964078~2902716,112016011~7352303,125863971~6236078,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,338235024~12677867,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.464 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,78964078~2902716,112016011~7352303,125863971~6236078,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,338235024~12677867,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.470 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,78964078~2902716,109051904~10316410,125829120~6270929,152168489~2837949,184198593~2978290,196098904~6602878,217769060~4997674,246409527~7192018,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,336640174~14272717,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.470 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 8 diff [4066564~13017605,22470476~1110648,32116244~7792378,61261300~9016548,77367094~4499700,112016011~7352303,125863971~6236078,152168489~2837949,164569697~386756,184198593~2978290,196098904~10224256,217769060~4997674,246409527~7192018,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,338235024~12677867,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.470 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,32116244~7792378,61261300~9016548,77367094~4499700,112016011~7352303,125863971~6236078,152168489~2837949,164569697~386756,184198593~2978290,196098904~10224256,217769060~4997674,246409527~7192018,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,338235024~12677867,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.476 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,32116244~7792378,61261300~9016548,77367094~4499700,109051904~10316410,125829120~6270929,152168489~2837949,164569697~386756,184198593~2978290,196098904~10224256,217769060~4997674,246409527~7192018,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,336640174~14272717,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.476 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,60560864~9716984,77367094~4499700,96669846~4086612,112016011~7352303,125863971~6236078,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,246409527~7192018,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,323712297~3134989,338235024~12677867,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.476 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,60560864~9716984,77367094~4499700,96669846~4086612,112016011~7352303,125863971~6236078,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,246409527~7192018,273362189~2881329,283756946~2345559,314923259~1047132,318803882~1606752,323712297~3134989,338235024~12677867,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.483 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,60560864~9716984,77367094~4499700,96669846~4086612,109051904~10316410,125829120~6270929,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,246409527~7192018,272629760~3613758,281018368~5084137,314923259~1047132,318803882~1606752,323712297~3134989,336640174~14272717,378654936~2811140,381681664~5724085,396077477~1902872] 2024-04-26T05:01:46.483 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,60560864~9716984,77367094~4499700,84186761~1603742,96669846~4086612,112016011~7352303,125863971~6236078,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,233204828~7303640,246409527~7192018,273362189~2881329,283756946~2345559,311409106~471389,314923259~1047132,316749913~3660721,323712297~3134989,338235024~12677867,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.483 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,60560864~9716984,77367094~4499700,84186761~1603742,96669846~4086612,112016011~7352303,125863971~6236078,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,233204828~7303640,246409527~7192018,273362189~2881329,283756946~2345559,311409106~471389,314923259~1047132,316749913~3660721,323712297~3134989,338235024~12677867,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.489 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,60560864~9716984,77367094~4499700,84186761~1603742,96669846~4086612,109051904~10316410,125829120~6270929,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,217769060~4997674,233106694~7401774,246409527~7192018,272629760~3613758,281018368~5084137,311409106~471389,314923259~1047132,316749913~3660721,323712297~3134989,336640174~14272717,356515840~8133627,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.489 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,77367094~4499700,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,112016011~7352303,125863971~6236078,141777995~7228088,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~6185885,233204828~7303640,246409527~7192018,273362189~2881329,282785062~3317443,311409106~471389,314923259~1047132,316749913~3660721,323712297~3134989,338235024~12677867,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.489 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,77367094~4499700,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,112016011~7352303,125863971~6236078,141777995~7228088,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~6185885,233204828~7303640,246409527~7192018,273362189~2881329,282785062~3317443,311409106~471389,314923259~1047132,316749913~3660721,323712297~3134989,338235024~12677867,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.496 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,60560864~9716984,77367094~4499700,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,109051904~10316410,125829120~6270929,141777995~7228088,152168489~2837949,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~6185885,233106694~7401774,246409527~7192018,272629760~3613758,281018368~5084137,311409106~471389,314923259~1047132,316749913~3660721,323712297~3134989,336640174~14272717,356515840~8133627,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.496 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,112016011~7352303,125863971~6236078,141777995~7228088,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,338235024~12677867,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.496 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,112016011~7352303,125863971~6236078,141777995~7228088,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,338235024~12677867,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.502 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,109051904~10316410,125829120~6270929,141777995~7228088,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,264379302~700168,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,336640174~14272717,356515840~8133627,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.502 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101273361~3260501,112016011~7352303,125863971~6236078,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,338235024~12677867,359690727~4958740,364764296~4408208,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.502 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101273361~3260501,112016011~7352303,125863971~6236078,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,338235024~12677867,359690727~4958740,364764296~4408208,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.508 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101033350~3500512,109051904~10316410,125829120~6270929,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,164569697~386756,166755333~5292336,184198593~2978290,191708486~14614674,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,264379302~700168,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,336640174~14272717,356515840~8133627,364764296~4408208,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.508 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,112016011~7352303,125863971~6236078,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.508 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,112016011~7352303,125863971~6236078,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,335899505~15013386,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.514 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101033350~7659632,109051904~10316410,125829120~6270929,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~3631573,191708486~14614674,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,264379302~700168,272629760~3613758,281018368~5084137,288524467~12377553,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,335899505~15013386,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.514 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,112016011~7352303,125863971~9109408,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.514 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,112016011~7352303,125863971~9109408,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,217239812~11461687,233204828~7303640,246080080~7521465,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.521 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~7659632,109051904~10316410,125829120~9144259,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,184198593~2978290,188005676~18317484,208442097~5429329,217239812~11461687,233106694~7401774,246080080~7521465,264379302~700168,272629760~3613758,281018368~5084137,288524467~20423730,311409106~471389,314923259~1047132,316749913~3660721,323712297~10855489,335899505~15013386,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.521 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 16 diff [814559~20471969,22470476~23558824,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~18279068,125863971~9109408,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~7521465,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,323712297~11379622,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.521 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~18279068,125863971~9109408,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~7521465,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,323712297~11379622,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.527 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~18334964,125829120~9144259,139237830~2273604,141777995~7228088,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~7521465,264379302~700168,268485267~2543234,272629760~3613758,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,323712297~11379622,335899505~15013386,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.527 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 17 diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~18279068,125863971~9109408,137230096~11775987,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,323712297~11379622,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.527 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~18279068,125863971~9109408,137230096~11775987,152168489~2837949,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,323712297~11379622,335899505~15013386,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.533 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~18334964,125829120~9144259,137230096~11775987,152168489~2837949,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~2543234,272629760~3613758,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,323712297~11379622,335899505~15013386,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.533 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 18 diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,323712297~27200594,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.533 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~24302884,314923259~1047132,316749913~3660721,323712297~27200594,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.539 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~24302884,314923259~1047132,316749913~3660721,323712297~27200594,352321536~16850968,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.539 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 19 diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.539 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.545 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.545 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 7 diff [9035947~8048222,125863971~6236078,273362189~2881329,283756946~2345559,318803882~1606752,338235024~5077544,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.546 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [9035947~8048222,125863971~6236078,273362189~2881329,283756946~2345559,318803882~1606752,338235024~5077544,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.551 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8695561,125829120~6270929,272629760~3613758,281018368~5084137,318803882~1606752,336640174~6672394,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.551 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 8 diff [4066564~13017605,22470476~1110648,32116244~7792378,61261300~9016548,77367094~2471889,125863971~6236078,164569697~386756,199723975~6599185,273362189~2881329,283756946~2345559,318803882~1606752,338235024~5077544,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.552 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~13017605,22470476~1110648,32116244~7792378,61261300~9016548,77367094~2471889,125863971~6236078,164569697~386756,199723975~6599185,273362189~2881329,283756946~2345559,318803882~1606752,338235024~5077544,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.557 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~17084169,20971520~2609604,32116244~7792378,61261300~9016548,77367094~2471889,125829120~6270929,164569697~386756,199723975~6599185,272629760~3613758,281018368~5084137,318803882~1606752,336640174~6672394,346214060~4698831,378654936~2811140,396077477~1902872] 2024-04-26T05:01:46.557 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,60560864~9716984,77367094~2471889,96669846~4086612,125863971~6236078,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,273362189~2881329,283756946~2345559,318803882~1606752,323712297~3134989,338235024~5077544,346214060~4698831,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.558 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,60560864~9716984,77367094~2471889,96669846~4086612,125863971~6236078,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,273362189~2881329,283756946~2345559,318803882~1606752,323712297~3134989,338235024~5077544,346214060~4698831,378654936~2811140,382845155~4560594,396077477~1902872] 2024-04-26T05:01:46.563 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,60560864~9716984,77367094~2471889,96669846~4086612,125829120~6270929,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,272629760~3613758,281018368~5084137,318803882~1606752,323712297~3134989,336640174~6672394,346214060~4698831,378654936~2811140,381681664~5724085,396077477~1902872] 2024-04-26T05:01:46.564 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,96669846~4086612,115339083~475404,125863971~6236078,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,233204828~7303640,273362189~2881329,283756946~2345559,311409106~471389,316749913~3660721,323712297~3134989,338235024~5077544,346214060~4698831,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.564 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,96669846~4086612,115339083~475404,125863971~6236078,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,233204828~7303640,273362189~2881329,283756946~2345559,311409106~471389,316749913~3660721,323712297~3134989,338235024~5077544,346214060~4698831,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.570 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,96669846~4086612,115339083~475404,125829120~6270929,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,233106694~7401774,272629760~3613758,281018368~5084137,311409106~471389,316749913~3660721,323712297~3134989,336640174~6672394,346214060~4698831,356515840~8133627,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.570 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,125863971~6236078,141777995~7228088,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,273362189~2881329,282785062~3317443,311409106~471389,316749913~3660721,323712297~3134989,338235024~5077544,346214060~4698831,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.570 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,125863971~6236078,141777995~7228088,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,273362189~2881329,282785062~3317443,311409106~471389,316749913~3660721,323712297~3134989,338235024~5077544,346214060~4698831,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.576 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,115339083~475404,125829120~6270929,141777995~7228088,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~6185885,233106694~7401774,248555976~3677622,272629760~3613758,281018368~5084137,311409106~471389,316749913~3660721,323712297~3134989,336640174~6672394,346214060~4698831,356515840~8133627,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.576 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,125863971~6236078,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,316749913~3660721,323712297~10855489,338235024~5077544,346214060~4698831,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.576 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,125863971~6236078,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,316749913~3660721,323712297~10855489,338235024~5077544,346214060~4698831,359690727~4958740,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.582 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,115339083~475404,125829120~6270929,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,316749913~3660721,323712297~10855489,336640174~6672394,346214060~4698831,356515840~8133627,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.582 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101273361~3260501,115339083~475404,125863971~6236078,139237830~2273604,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,316749913~3660721,323712297~10855489,338235024~5077544,346214060~4698831,359690727~4958740,364764296~4408208,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.582 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101273361~3260501,115339083~475404,125863971~6236078,139237830~2273604,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273362189~2881329,282785062~3317443,288524467~3605886,294153965~6748055,311409106~471389,316749913~3660721,323712297~10855489,338235024~5077544,346214060~4698831,359690727~4958740,364764296~4408208,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.588 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101033350~3500512,115339083~475404,125829120~6270929,139237830~2273604,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,272629760~3613758,281018368~5084137,288524467~3605886,293601280~7300740,311409106~471389,316749913~3660721,323712297~10855489,336640174~6672394,346214060~4698831,356515840~8133627,364764296~4408208,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.588 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,115339083~475404,125863971~6236078,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~3631573,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,316749913~3660721,323712297~10855489,335899505~7413063,346214060~4698831,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.588 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,115339083~475404,125863971~6236078,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~3631573,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~12377553,311409106~471389,316749913~3660721,323712297~10855489,335899505~7413063,346214060~4698831,359690727~9481777,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.594 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101033350~7659632,115339083~475404,125829120~6270929,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,188005676~3631573,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,272629760~3613758,281018368~5084137,288524467~12377553,311409106~471389,316749913~3660721,323712297~10855489,335899505~7413063,346214060~4698831,356515840~12656664,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.594 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,115339083~475404,125863971~9109408,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~10774889,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,316749913~3660721,323712297~10855489,335899505~7413063,346214060~4698831,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.594 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,115339083~475404,125863971~9109408,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~10774889,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,273362189~2881329,282785062~3317443,288524467~20423730,311409106~471389,316749913~3660721,323712297~10855489,335899505~7413063,346214060~4698831,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.600 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~7659632,115339083~475404,125829120~9144259,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,188005676~10774889,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,272629760~3613758,281018368~5084137,288524467~20423730,311409106~471389,316749913~3660721,323712297~10855489,335899505~7413063,346214060~4698831,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.600 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 16 diff [814559~20471969,22470476~23558824,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,125863971~9109408,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,316749913~3660721,323712297~11379622,335899505~7413063,346214060~4698831,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.600 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,125863971~9109408,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,316749913~3660721,323712297~11379622,335899505~7413063,346214060~4698831,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.606 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~15153106,125829120~9144259,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,272629760~3613758,281018368~5084137,288524467~24302884,316749913~3660721,323712297~11379622,335899505~7413063,346214060~4698831,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.606 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 17 diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,125863971~9109408,137230096~11775987,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,316749913~3660721,323712297~11379622,335899505~7413063,346214060~4698831,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.606 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,125863971~9109408,137230096~11775987,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,273362189~2881329,282785062~3317443,288524467~24302884,316749913~3660721,323712297~11379622,335899505~7413063,346214060~4698831,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.613 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~15153106,125829120~9144259,137230096~11775987,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,272629760~3613758,281018368~5084137,288524467~24302884,316749913~3660721,323712297~11379622,335899505~7413063,346214060~4698831,356515840~12656664,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.613 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 18 diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15194611,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~24302884,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.613 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15194611,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~24302884,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.619 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15194611,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~24302884,316749913~3660721,323712297~19600271,346214060~4698831,352321536~16850968,369299132~1019592,371817617~5119593,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.619 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 19 diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.620 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:46.626 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:46.626 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 8 diff [4066564~6157893,22470476~1110648,32116244~7792378,61261300~9016548,77367094~2471889,164569697~386756,199723975~6599185,397306655~47324] 2024-04-26T05:01:46.626 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~6157893,22470476~1110648,32116244~7792378,61261300~9016548,77367094~2471889,164569697~386756,199723975~6599185,397306655~47324] 2024-04-26T05:01:46.633 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~10224457,20971520~2609604,32116244~7792378,61261300~9016548,77367094~2471889,164569697~386756,199723975~6599185,397306655~47324] 2024-04-26T05:01:46.633 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 9 diff [4066564~17219964,22470476~1110648,32116244~7792378,60560864~9716984,77367094~2471889,96669846~4086612,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,323712297~3134989,382845155~4560594,397306655~47324] 2024-04-26T05:01:46.633 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~1110648,32116244~7792378,60560864~9716984,77367094~2471889,96669846~4086612,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,323712297~3134989,382845155~4560594,397306655~47324] 2024-04-26T05:01:46.639 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~23581124,32116244~7792378,60560864~9716984,77367094~2471889,96669846~4086612,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,323712297~3134989,381681664~5724085,397306655~47324] 2024-04-26T05:01:46.639 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 10 diff [4066564~17219964,22470476~4781207,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,96669846~4086612,115339083~475404,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,233204828~7303640,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.639 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~4781207,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,96669846~4086612,115339083~475404,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,233204828~7303640,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.645 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~27251683,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,96669846~4086612,115339083~475404,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,233106694~7401774,311409106~471389,316749913~2883833,323712297~3134989,356515840~8133627,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.645 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 11 diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.645 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.651 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,60560864~9716984,77367094~2471889,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,115339083~475404,141777995~7228088,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~6185885,233106694~7401774,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,323712297~3134989,356515840~8133627,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.651 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 12 diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.652 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4066564~17219964,22470476~7398355,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.657 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,60560864~9716984,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,115339083~475404,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,311409106~471389,316749913~2883833,323712297~10855489,356515840~8133627,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.657 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 13 diff [814559~20471969,22470476~7398355,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101273361~3260501,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,364764296~4408208,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.657 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101273361~3260501,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,364764296~4408208,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.663 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~7792378,56710724~189312,60560864~9716984,76265946~12315675,96669846~4086612,101033350~3500512,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,164569697~386756,166755333~5292336,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,311409106~471389,316749913~2883833,323712297~10855489,356515840~8133627,364764296~4408208,371817617~5119593,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.663 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 14 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~3631573,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.663 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~3631573,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.669 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,56710724~189312,60560864~9716984,73504814~1175543,76265946~12315675,96669846~4086612,101033350~7659632,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,188005676~3631573,191708486~7072079,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,356515840~12656664,371817617~5119593,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.669 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 15 diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~10774889,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.669 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~7398355,32116244~7792378,40504854~4916540,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,188005676~10774889,199723975~6599185,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.676 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29868831,32116244~13305150,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~7659632,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,188005676~10774889,199723975~6599185,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.676 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 16 diff [814559~20471969,22470476~23558824,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.676 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.681 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,52116421~513575,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~15153106,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.681 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 17 diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.681 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.688 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14586755,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.688 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 18 diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,323712297~19388386,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.688 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,323712297~19388386,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.694 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,323712297~19388386,352321536~16850968,369299132~1019592,371817617~5119593,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.694 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 19 diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.694 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:46.700 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:46.700 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 9 diff [10138532~11147996,60560864~6873295,96669846~4086612,166755333~5292336,191708486~7072079,323712297~3134989,382845155~4560594] 2024-04-26T05:01:46.700 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [10138532~11147996,60560864~6873295,96669846~4086612,166755333~5292336,191708486~7072079,323712297~3134989,382845155~4560594] 2024-04-26T05:01:46.706 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [10138532~11147996,60560864~6873295,96669846~4086612,166755333~5292336,191708486~7072079,323712297~3134989,381681664~5724085] 2024-04-26T05:01:46.706 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 10 diff [10138532~11147996,22694932~4556751,60560864~7148987,84186761~1603742,96669846~4086612,115339083~475404,166755333~5292336,191708486~7072079,233204828~7303640,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214] 2024-04-26T05:01:46.706 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [10138532~11147996,22694932~4556751,60560864~7148987,84186761~1603742,96669846~4086612,115339083~475404,166755333~5292336,191708486~7072079,233204828~7303640,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214] 2024-04-26T05:01:46.712 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [10138532~11147996,22694932~4556751,60560864~7148987,84186761~1603742,96669846~4086612,115339083~475404,166755333~5292336,191708486~7072079,233106694~7401774,311409106~471389,316749913~2883833,323712297~3134989,356515840~8133627,381681664~8494705] 2024-04-26T05:01:46.712 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 11 diff [10138532~11147996,22694932~7173899,34972837~4732990,60560864~7148987,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,166755333~5292336,191708486~7072079,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214] 2024-04-26T05:01:46.712 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [10138532~11147996,22694932~7173899,34972837~4732990,60560864~7148987,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,166755333~5292336,191708486~7072079,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,323712297~3134989,359690727~4958740,382845155~7331214] 2024-04-26T05:01:46.718 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [10138532~11147996,22694932~7173899,34972837~4732990,60560864~7148987,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,115339083~475404,141777995~7228088,166755333~5292336,191708486~7072079,208442097~5429329,217239812~6185885,233106694~7401774,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,323712297~3134989,356515840~8133627,381681664~8494705] 2024-04-26T05:01:46.718 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 12 diff [10138532~11147996,22694932~7173899,34972837~4732990,60560864~7148987,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,156533592~1041666,166755333~5292336,191708486~7072079,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,382845155~7331214] 2024-04-26T05:01:46.718 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [10138532~11147996,22694932~7173899,34972837~4732990,60560864~7148987,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101273361~3260501,115339083~475404,141777995~7228088,156533592~1041666,166755333~5292336,191708486~7072079,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,382845155~7331214] 2024-04-26T05:01:46.724 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [10138532~11147996,22694932~7173899,34972837~4732990,60560864~7148987,76265946~6615286,84186761~1603742,86520737~1359758,96669846~4086612,101033350~3500512,115339083~475404,141777995~7228088,156533592~1041666,166755333~5292336,191708486~7072079,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,311409106~471389,316749913~2883833,323712297~10855489,356515840~8133627,381681664~8494705] 2024-04-26T05:01:46.724 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 13 diff [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,56710724~189312,60560864~9194932,76265946~12315675,96669846~4086612,101273361~3260501,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,166755333~5292336,191708486~7072079,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,364764296~4408208,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.724 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,56710724~189312,60560864~9194932,76265946~12315675,96669846~4086612,101273361~3260501,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,166755333~5292336,191708486~7072079,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,323712297~10855489,359690727~4958740,364764296~4408208,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.730 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,56710724~189312,60560864~9194932,76265946~12315675,96669846~4086612,101033350~3500512,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,166755333~5292336,191708486~7072079,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,311409106~471389,316749913~2883833,323712297~10855489,356515840~8133627,364764296~4408208,371817617~5119593,381681664~8494705] 2024-04-26T05:01:46.730 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 14 diff [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,56710724~189312,60560864~9194932,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,166755333~5292336,188005676~3631573,191708486~7072079,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.731 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,56710724~189312,60560864~9194932,73504814~1175543,76265946~12315675,96669846~4086612,101089246~7603736,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,166755333~5292336,188005676~3631573,191708486~7072079,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.737 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,39908622~5512772,56710724~189312,60560864~9194932,73504814~1175543,76265946~12315675,96669846~4086612,101033350~7659632,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,166755333~5292336,188005676~3631573,191708486~7072079,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,356515840~12656664,371817617~5119593,381681664~8494705] 2024-04-26T05:01:46.737 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 15 diff [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,52116421~513575,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,166755333~5292336,188005676~10774889,200091224~4513414,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.737 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,52116421~513575,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101089246~7603736,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,166755333~5292336,188005676~10774889,200091224~4513414,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.743 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,10138532~11147996,22694932~7173899,33990443~728587,34972837~4732990,39908622~5512772,52116421~513575,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101033350~7659632,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,166755333~5292336,188005676~10774889,200091224~4513414,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,323712297~10855489,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,381681664~8494705] 2024-04-26T05:01:46.743 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 16 diff [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.743 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.749 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101033350~15153106,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,381681664~8494705] 2024-04-26T05:01:46.749 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 17 diff [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.749 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.755 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14586755,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,323712297~11379622,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,381681664~8494705] 2024-04-26T05:01:46.755 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 18 diff [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,323712297~19388386,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.755 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,323712297~19388386,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,382845155~7331214] 2024-04-26T05:01:46.761 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,323712297~19388386,352321536~16850968,369299132~1019592,371817617~5119593,381681664~8494705] 2024-04-26T05:01:46.761 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 19 diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:46.761 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:46.767 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:46.767 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 10 diff [16659593~1748287,22694932~4556751,64260123~3449728,84186761~1603742,115339083~475404,233204828~7303640,311409106~471389,316749913~2883833,359690727~4958740,383510826~6665543] 2024-04-26T05:01:46.767 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [16659593~1748287,22694932~4556751,64260123~3449728,84186761~1603742,115339083~475404,233204828~7303640,311409106~471389,316749913~2883833,359690727~4958740,383510826~6665543] 2024-04-26T05:01:46.773 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [16659593~1748287,22694932~4556751,64260123~3449728,84186761~1603742,115339083~475404,233106694~7401774,311409106~471389,316749913~2883833,356515840~8133627,383510826~6665543] 2024-04-26T05:01:46.773 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 11 diff [16659593~1748287,22694932~7173899,34972837~4732990,64260123~3449728,84186761~1603742,86520737~1359758,101273361~3260501,115339083~475404,141777995~7228088,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,359690727~4958740,383510826~6665543] 2024-04-26T05:01:46.773 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [16659593~1748287,22694932~7173899,34972837~4732990,64260123~3449728,84186761~1603742,86520737~1359758,101273361~3260501,115339083~475404,141777995~7228088,208442097~5429329,217239812~6185885,233204828~7303640,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,359690727~4958740,383510826~6665543] 2024-04-26T05:01:46.779 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [16659593~1748287,22694932~7173899,34972837~4732990,64260123~3449728,84186761~1603742,86520737~1359758,101033350~3500512,115339083~475404,141777995~7228088,208442097~5429329,217239812~6185885,233106694~7401774,248555976~3677622,282785062~2730194,311409106~471389,316749913~2883833,356515840~8133627,383510826~6665543] 2024-04-26T05:01:46.780 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 12 diff [16659593~1748287,22694932~7173899,34972837~4732990,64260123~3449728,76265946~6615286,84186761~1603742,86520737~1359758,101273361~3260501,115339083~475404,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,326595993~7971793,359690727~4958740,383510826~6665543] 2024-04-26T05:01:46.780 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [16659593~1748287,22694932~7173899,34972837~4732990,64260123~3449728,76265946~6615286,84186761~1603742,86520737~1359758,101273361~3260501,115339083~475404,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,326595993~7971793,359690727~4958740,383510826~6665543] 2024-04-26T05:01:46.785 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [16659593~1748287,22694932~7173899,34972837~4732990,64260123~3449728,76265946~6615286,84186761~1603742,86520737~1359758,101033350~3500512,115339083~475404,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,311409106~471389,316749913~2883833,326595993~7971793,356515840~8133627,383510826~6665543] 2024-04-26T05:01:46.785 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 13 diff [814559~6704083,16659593~1748287,22694932~7173899,33990443~728587,34972837~4732990,56710724~189312,63000434~6755362,76265946~12315675,101273361~3260501,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,326595993~7971793,359690727~4958740,364764296~4408208,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.786 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,16659593~1748287,22694932~7173899,33990443~728587,34972837~4732990,56710724~189312,63000434~6755362,76265946~12315675,101273361~3260501,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,311409106~471389,316749913~2883833,326595993~7971793,359690727~4958740,364764296~4408208,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.791 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,16659593~1748287,22694932~7173899,33990443~728587,34972837~4732990,56710724~189312,63000434~6755362,76265946~12315675,101033350~3500512,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,311409106~471389,316749913~2883833,326595993~7971793,356515840~8133627,364764296~4408208,371817617~5119593,383510826~6665543] 2024-04-26T05:01:46.791 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 14 diff [814559~6704083,16659593~1748287,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~3631573,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,326595993~7971793,335899505~7201178,359690727~9481777,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.791 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,16659593~1748287,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~3631573,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,326595993~7971793,335899505~7201178,359690727~9481777,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.797 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,16659593~1748287,22694932~7173899,33990443~728587,34972837~4732990,39908622~5512772,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101033350~7659632,115339083~475404,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,188005676~3631573,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,311409106~471389,316749913~2883833,326595993~7971793,335899505~7201178,356515840~12656664,371817617~5119593,383510826~6665543] 2024-04-26T05:01:46.798 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 15 diff [814559~6704083,12820204~6774326,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~9582010,200091224~4513414,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,326595993~7971793,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.798 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,22694932~7173899,33990443~728587,34972837~4732990,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~9582010,200091224~4513414,208442097~5429329,217239812~11461687,233204828~7303640,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,326595993~7971793,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.803 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,22694932~7173899,33990443~728587,34972837~4732990,39908622~5512772,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101033350~7659632,115339083~475404,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,188005676~9582010,200091224~4513414,208442097~5429329,217239812~11461687,233106694~7401774,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,311409106~471389,316749913~2883833,326595993~7971793,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,383510826~6665543] 2024-04-26T05:01:46.803 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 16 diff [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,325766628~9325291,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.803 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,325766628~9325291,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.809 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101033350~15153106,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~7887016,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,325766628~9325291,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,383510826~6665543] 2024-04-26T05:01:46.809 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 17 diff [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,325766628~9325291,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.809 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,325766628~9325291,335899505~7201178,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.816 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101033350~15153106,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,316749913~2883833,325766628~9325291,335899505~7201178,356515840~12656664,369299132~1019592,371817617~5119593,383510826~6665543] 2024-04-26T05:01:46.816 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 18 diff [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,325766628~17334055,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.816 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,325766628~17334055,355398337~3388765,359690727~9481777,369299132~1019592,372335201~4602009,383510826~6665543] 2024-04-26T05:01:46.822 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,316749913~2883833,325766628~17334055,352321536~16850968,369299132~1019592,371817617~5119593,383510826~6665543] 2024-04-26T05:01:46.822 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 19 diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:46.822 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:46.828 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:46.828 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 11 diff [25627621~4241210,34972837~4732990,86520737~1359758,101273361~3260501,141777995~7228088,208442097~5429329,217239812~6185885,248555976~3677622,282785062~2730194] 2024-04-26T05:01:46.828 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [25627621~4241210,34972837~4732990,86520737~1359758,101273361~3260501,141777995~7228088,208442097~5429329,217239812~6185885,248555976~3677622,282785062~2730194] 2024-04-26T05:01:46.834 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [25627621~4241210,34972837~4732990,86520737~1359758,101033350~3500512,141777995~7228088,208442097~5429329,217239812~6185885,248555976~3677622,282785062~2730194] 2024-04-26T05:01:46.834 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 12 diff [25627621~4241210,34972837~4732990,76265946~6615286,86520737~1359758,101273361~3260501,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,326595993~7971793] 2024-04-26T05:01:46.834 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [25627621~4241210,34972837~4732990,76265946~6615286,86520737~1359758,101273361~3260501,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,326595993~7971793] 2024-04-26T05:01:46.840 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [25627621~4241210,34972837~4732990,76265946~6615286,86520737~1359758,101033350~3500512,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,326595993~7971793] 2024-04-26T05:01:46.840 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 13 diff [814559~6704083,25627621~4241210,33990443~728587,34972837~4732990,56710724~189312,63000434~6755362,76265946~12315675,101273361~3260501,139237830~2273604,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,326595993~7971793,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.840 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,25627621~4241210,33990443~728587,34972837~4732990,56710724~189312,63000434~6755362,76265946~12315675,101273361~3260501,139237830~2273604,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,294153965~6748055,326595993~7971793,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.846 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,25627621~4241210,33990443~728587,34972837~4732990,56710724~189312,63000434~6755362,76265946~12315675,101033350~3500512,139237830~2273604,141777995~7228088,156533592~1041666,208442097~5429329,217239812~11461687,246080080~6153518,264379302~700168,282785062~2730194,288524467~3605886,293601280~7300740,326595993~7971793,362613635~1366061,364764296~4408208,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.846 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 14 diff [814559~6704083,25627621~4241210,33990443~728587,34972837~4732990,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~3631573,208442097~5429329,217239812~11461687,236032731~2222755,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,326595993~7971793,335899505~7201178,360401098~8771406,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.847 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,25627621~4241210,33990443~728587,34972837~4732990,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~3631573,208442097~5429329,217239812~11461687,236032731~2222755,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,326595993~7971793,335899505~7201178,360401098~8771406,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.852 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,25627621~4241210,33990443~728587,34972837~4732990,39908622~5512772,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101033350~7659632,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,188005676~3631573,208442097~5429329,217239812~11461687,236032731~2222755,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~12377553,326595993~7971793,335899505~7201178,360401098~8771406,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.852 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 15 diff [814559~6704083,12820204~6774326,25627621~4241210,33990443~728587,34972837~4732990,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~9582010,200091224~4513414,208442097~5429329,217239812~11461687,236032731~2222755,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,326595993~7971793,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.853 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,25627621~4241210,33990443~728587,34972837~4732990,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,188005676~9582010,200091224~4513414,208442097~5429329,217239812~11461687,236032731~2222755,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,326595993~7971793,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.858 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,25627621~4241210,33990443~728587,34972837~4732990,39908622~5512772,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101033350~7659632,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,188005676~9582010,200091224~4513414,208442097~5429329,217239812~11461687,236032731~2222755,246080080~6153518,264379302~700168,273204716~24336,282785062~2730194,288524467~20423730,326595993~7971793,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.858 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 16 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~5634034,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.858 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~5634034,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.864 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101033350~15153106,127854997~7118382,139237830~2273604,141777995~7228088,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,208442097~5429329,217239812~11461687,232621452~5634034,246080080~6153518,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.864 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 17 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.864 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.870 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101033350~15153106,126499545~375794,127854997~7118382,137230096~11775987,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,282785062~2730194,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.870 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 18 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,325766628~17334055,355398337~3388765,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.870 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,325766628~17334055,355398337~3388765,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.876 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15194611,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,206811004~7060422,217239812~11461687,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~24302884,325766628~17334055,352321536~6465566,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.876 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 19 diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.876 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.882 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.883 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 12 diff [76265946~6615286,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,294153965~6748055,326595993~7971793] 2024-04-26T05:01:46.883 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [76265946~6615286,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,294153965~6748055,326595993~7971793] 2024-04-26T05:01:46.889 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [76265946~6615286,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,293601280~7300740,326595993~7971793] 2024-04-26T05:01:46.889 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 13 diff [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,76265946~12315675,139237830~2273604,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,294153965~6748055,326595993~7971793,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.889 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,76265946~12315675,139237830~2273604,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,294153965~6748055,326595993~7971793,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.895 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,76265946~12315675,139237830~2273604,156533592~1041666,221093468~7608031,246080080~3588531,264379302~700168,288524467~3605886,293601280~7300740,326595993~7971793,362613635~1366061,364764296~4408208,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.895 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 14 diff [814559~6704083,33990443~728587,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,139237830~2273604,156533592~1041666,161552797~233185,188005676~3631573,221093468~7608031,236032731~2222755,246080080~3588531,264379302~700168,273204716~24336,288524467~12377553,326595993~7971793,335899505~7201178,360401098~8771406,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.895 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,33990443~728587,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,139237830~2273604,156533592~1041666,161552797~233185,188005676~3631573,221093468~7608031,236032731~2222755,246080080~3588531,264379302~700168,273204716~24336,288524467~12377553,326595993~7971793,335899505~7201178,360401098~8771406,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.900 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,33990443~728587,39908622~5512772,56710724~189312,63000434~6755362,73504814~1175543,76265946~12315675,101089246~7603736,139237830~2273604,156533592~1041666,159383552~2402430,188005676~3631573,221093468~7608031,236032731~2222755,246080080~3588531,264379302~700168,273204716~24336,288524467~12377553,326595993~7971793,335899505~7201178,360401098~8771406,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.901 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 15 diff [814559~6704083,12820204~6774326,33990443~728587,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,127854997~7118382,139237830~2273604,156533592~1041666,161552797~233185,188005676~9582010,200091224~4513414,221093468~7608031,236032731~2222755,246080080~3588531,264379302~700168,273204716~24336,288524467~20423730,326595993~7971793,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.901 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,33990443~728587,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,127854997~7118382,139237830~2273604,156533592~1041666,161552797~233185,188005676~9582010,200091224~4513414,221093468~7608031,236032731~2222755,246080080~3588531,264379302~700168,273204716~24336,288524467~20423730,326595993~7971793,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.907 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,33990443~728587,39908622~5512772,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~7603736,127854997~7118382,139237830~2273604,156533592~1041666,159383552~2402430,188005676~9582010,200091224~4513414,221093468~7608031,236032731~2222755,246080080~3588531,264379302~700168,273204716~24336,288524467~20423730,326595993~7971793,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.907 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 16 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,221093468~7608031,232621452~5634034,246080080~3588531,264379302~700168,268485267~2543234,273204716~24336,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.908 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,221093468~7608031,232621452~5634034,246080080~3588531,264379302~700168,268485267~2543234,273204716~24336,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.914 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,127854997~7118382,139237830~2273604,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,221093468~7608031,232621452~5634034,246080080~3588531,264379302~700168,268485267~2543234,273204716~24336,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.914 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 17 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.914 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.920 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14586755,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,206811004~5690052,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~2543234,273204716~24336,288524467~24302884,325766628~9325291,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.920 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 18 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~24302884,325766628~17334055,355398337~3388765,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.920 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,156533592~1041666,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~24302884,325766628~17334055,355398337~3388765,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.926 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,156533592~1041666,159383552~2402430,180999320~6640455,188005676~17072720,206811004~5690052,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~24302884,325766628~17334055,352321536~6465566,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.926 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 19 diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.926 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.932 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.932 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 13 diff [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,82879792~5701829,139237830~2273604,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.932 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,82879792~5701829,139237830~2273604,362613635~1366061,364764296~4408208,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.938 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,33990443~728587,56710724~189312,63000434~6755362,82879792~5701829,139237830~2273604,362613635~1366061,364764296~4408208,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.938 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 14 diff [814559~6704083,33990443~728587,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,82879792~5701829,101089246~7603736,139237830~2273604,161552797~233185,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~8771406,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.938 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,33990443~728587,40504854~4916540,56710724~189312,63000434~6755362,73504814~1175543,82879792~5701829,101089246~7603736,139237830~2273604,161552797~233185,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~8771406,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.944 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,33990443~728587,39908622~5512772,56710724~189312,63000434~6755362,73504814~1175543,82879792~5701829,101089246~7603736,139237830~2273604,159383552~2402430,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~8771406,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.944 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 15 diff [814559~6704083,12820204~6774326,33990443~728587,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~7603736,127854997~7118382,139237830~2273604,161552797~233185,188005676~9582010,200091224~4513414,236032731~2222755,273204716~24336,290301230~6519424,299789837~9158360,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.944 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,33990443~728587,40504854~4916540,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~7603736,127854997~7118382,139237830~2273604,161552797~233185,188005676~9582010,200091224~4513414,236032731~2222755,273204716~24336,290301230~6519424,299789837~9158360,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.950 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,33990443~728587,39908622~5512772,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~7603736,127854997~7118382,139237830~2273604,159383552~2402430,188005676~9582010,200091224~4513414,236032731~2222755,273204716~24336,290301230~6519424,299789837~9158360,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.950 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 16 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~15097210,127854997~7118382,139237830~2273604,161552797~233185,180999320~6640455,188005676~17072720,232621452~5634034,268485267~2543234,273204716~24336,290301230~6519424,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.950 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~15097210,127854997~7118382,139237830~2273604,161552797~233185,180999320~6640455,188005676~17072720,232621452~5634034,268485267~2543234,273204716~24336,290301230~6519424,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.956 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,52116421~513575,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~15097210,127854997~7118382,139237830~2273604,159383552~2402430,180999320~6640455,188005676~17072720,232621452~5634034,268485267~2543234,273204716~24336,290301230~6519424,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.956 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 17 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,232621452~5634034,257797223~2913328,268485267~2543234,273204716~24336,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.956 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,232621452~5634034,257797223~2913328,268485267~2543234,273204716~24336,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.962 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~7972909,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,159383552~2402430,180999320~6640455,188005676~17072720,206811004~5690052,232621452~5634034,257797223~2913328,268485267~2543234,273204716~24336,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.962 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 18 diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~11033496,355398337~3388765,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.962 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~11033496,355398337~3388765,360401098~8771406,369299132~1019592,372335201~4602009,384421620~5746065] 2024-04-26T05:01:46.968 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6704083,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,159383552~2402430,180999320~6640455,188005676~17072720,206811004~5690052,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~11033496,352321536~6465566,360401098~8771406,369299132~1019592,371817617~5119593,384421620~5746065] 2024-04-26T05:01:46.968 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 19 diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.968 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.974 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:46.974 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 14 diff [40504854~4916540,73504814~1175543,101089246~7603736,161552797~233185,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~4874710] 2024-04-26T05:01:46.974 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [40504854~4916540,73504814~1175543,101089246~7603736,161552797~233185,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~4874710] 2024-04-26T05:01:46.980 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [39908622~5512772,73504814~1175543,101089246~7603736,159383552~2402430,188005676~3631573,236032731~2222755,273204716~24336,290301230~6519424,335899505~7201178,360401098~4874710] 2024-04-26T05:01:46.980 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 15 diff [12820204~6774326,40504854~4916540,52116421~513575,73504814~1175543,85238503~5614198,101089246~7603736,127854997~7118382,161552797~233185,188005676~9582010,200091224~4513414,236032731~2222755,273204716~24336,290301230~6519424,299789837~9158360,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.980 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,40504854~4916540,52116421~513575,73504814~1175543,85238503~5614198,101089246~7603736,127854997~7118382,161552797~233185,188005676~9582010,200091224~4513414,236032731~2222755,273204716~24336,290301230~6519424,299789837~9158360,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.986 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,39908622~5512772,52116421~513575,73504814~1175543,85238503~5614198,101089246~7603736,127854997~7118382,159383552~2402430,188005676~9582010,200091224~4513414,236032731~2222755,273204716~24336,290301230~6519424,299789837~9158360,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.986 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 16 diff [12820204~6774326,24844439~7388720,38082429~7946871,52116421~513575,73504814~1175543,85238503~5614198,101089246~15097210,127854997~7118382,161552797~233185,180999320~6640455,188005676~17072720,232621452~5634034,268485267~2543234,273204716~24336,290301230~6519424,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.986 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,24844439~7388720,38082429~7946871,52116421~513575,73504814~1175543,85238503~5614198,101089246~15097210,127854997~7118382,161552797~233185,180999320~6640455,188005676~17072720,232621452~5634034,268485267~2543234,273204716~24336,290301230~6519424,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.992 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,24844439~7388720,38082429~7946871,52116421~513575,73504814~1175543,85238503~5614198,101089246~15097210,127854997~7118382,159383552~2402430,180999320~6640455,188005676~17072720,232621452~5634034,268485267~2543234,273204716~24336,290301230~6519424,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.992 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 17 diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,73504814~1175543,85238503~5614198,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,232621452~5634034,257797223~2913328,268485267~2543234,273204716~24336,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.992 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,73504814~1175543,85238503~5614198,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,232621452~5634034,257797223~2913328,268485267~2543234,273204716~24336,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.998 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,73504814~1175543,85238503~5614198,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,159383552~2402430,180999320~6640455,188005676~17072720,206811004~5690052,232621452~5634034,257797223~2913328,268485267~2543234,273204716~24336,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,335899505~7201178,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.998 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 18 diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~11033496,355398337~3388765,360401098~5492302,369299132~1019592] 2024-04-26T05:01:46.998 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,161552797~233185,180999320~6640455,188005676~17072720,206811004~5690052,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~11033496,355398337~3388765,360401098~5492302,369299132~1019592] 2024-04-26T05:01:47.004 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,159383552~2402430,180999320~6640455,188005676~17072720,206811004~5690052,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~13037514,325766628~2902956,332067187~11033496,352321536~6465566,360401098~5492302,369299132~1019592] 2024-04-26T05:01:47.004 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 19 diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:47.004 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:47.010 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~12706788] 2024-04-26T05:01:47.010 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 15 diff [12820204~6774326,52116421~513575,85238503~5614198,127854997~7118382,190002781~7584905,200091224~4513414,299789837~9158360,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.010 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,52116421~513575,85238503~5614198,127854997~7118382,190002781~7584905,200091224~4513414,299789837~9158360,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.016 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,52116421~513575,85238503~5614198,127854997~7118382,190002781~7584905,200091224~4513414,299789837~9158360,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 16 diff [12820204~6774326,24844439~7388720,38082429~7946871,52116421~513575,85238503~5614198,108350706~7835750,127854997~7118382,180999320~6640455,190002781~15075615,232621452~5631968,268485267~2543234,299789837~13037514,325766628~2902956,332067187~3024732,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,24844439~7388720,38082429~7946871,52116421~513575,85238503~5614198,108350706~7835750,127854997~7118382,180999320~6640455,190002781~15075615,232621452~5631968,268485267~2543234,299789837~13037514,325766628~2902956,332067187~3024732,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.023 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,24844439~7388720,38082429~7946871,52116421~513575,85238503~5614198,108350706~7835750,127854997~7118382,180999320~6640455,190002781~15075615,232621452~5631968,268485267~2543234,299789837~13037514,325766628~2902956,332067187~3024732,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.023 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 17 diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,85238503~5614198,104551092~11635364,126499545~375794,127854997~7118382,137230096~4951286,180999320~6640455,190002781~15075615,206811004~5690052,232621452~5631968,257797223~2913328,268485267~2543234,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,337362700~1413194,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.023 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,85238503~5614198,104551092~11635364,126499545~375794,127854997~7118382,137230096~4951286,180999320~6640455,190002781~15075615,206811004~5690052,232621452~5631968,257797223~2913328,268485267~2543234,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,337362700~1413194,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,85238503~5614198,104551092~11635364,126499545~375794,127854997~7118382,137230096~4951286,180999320~6640455,190002781~15075615,206811004~5690052,232621452~5631968,257797223~2913328,268485267~2543234,297925974~110671,299789837~13037514,325766628~2902956,332067187~3024732,337362700~1413194,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 18 diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,180999320~6640455,190002781~15075615,206811004~5690052,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~13037514,325766628~2902956,332067187~10019772,355398337~3388765,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,180999320~6640455,190002781~15075615,206811004~5690052,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~13037514,325766628~2902956,332067187~10019772,355398337~3388765,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12820204~6774326,24844439~7388720,38082429~7946871,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~4951286,143863691~8561016,180999320~6640455,190002781~15075615,206811004~5690052,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~13037514,325766628~2902956,332067187~10019772,352321536~6465566,363806101~2087299,369299132~1019592] 2024-04-26T05:01:47.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 19 diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:47.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:47.041 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,352321536~6465566,363806101~9301785] 2024-04-26T05:01:47.041 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 16 diff [24844439~7388720,38082429~7946871,108350706~7835750,180999320~6640455,197058233~8020163,232621452~5631968,268485267~2543234,307050062~5777289,325766628~2902956,332067187~3024732] 2024-04-26T05:01:47.041 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [24844439~7388720,38082429~7946871,108350706~7835750,180999320~6640455,197058233~8020163,232621452~5631968,268485267~2543234,307050062~5777289,325766628~2902956,332067187~3024732] 2024-04-26T05:01:47.047 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [24844439~7388720,38082429~7946871,108350706~7835750,180999320~6640455,197058233~8020163,232621452~5631968,268485267~2543234,307050062~5777289,325766628~2902956,332067187~3024732] 2024-04-26T05:01:47.047 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 17 diff [24844439~7388720,38082429~7946871,50191347~5428687,104551092~11635364,126499545~375794,137230096~4951286,180999320~6640455,197058233~8020163,206811004~5690052,232621452~5631968,257797223~2913328,268485267~2543234,297925974~110671,307050062~5777289,325766628~2902956,332067187~3024732,337362700~1413194] 2024-04-26T05:01:47.048 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [24844439~7388720,38082429~7946871,50191347~5428687,104551092~11635364,126499545~375794,137230096~4951286,180999320~6640455,197058233~8020163,206811004~5690052,232621452~5631968,257797223~2913328,268485267~2543234,297925974~110671,307050062~5777289,325766628~2902956,332067187~3024732,337362700~1413194] 2024-04-26T05:01:47.054 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [24844439~7388720,38082429~7946871,50191347~5428687,104551092~11635364,126499545~375794,137230096~4951286,180999320~6640455,197058233~8020163,206811004~5690052,232621452~5631968,257797223~2913328,268485267~2543234,297925974~110671,307050062~5777289,325766628~2902956,332067187~3024732,337362700~1413194] 2024-04-26T05:01:47.054 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 18 diff [24844439~7388720,38082429~7946871,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~4951286,143863691~8561016,180999320~6640455,197058233~8020163,206811004~5690052,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~5777289,325766628~2902956,332067187~10019772,355398337~3388765] 2024-04-26T05:01:47.054 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [24844439~7388720,38082429~7946871,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~4951286,143863691~8561016,180999320~6640455,197058233~8020163,206811004~5690052,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~5777289,325766628~2902956,332067187~10019772,355398337~3388765] 2024-04-26T05:01:47.060 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [24844439~7388720,38082429~7946871,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~4951286,143863691~8561016,180999320~6640455,197058233~8020163,206811004~5690052,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~5777289,325766628~2902956,332067187~10019772,352321536~6465566] 2024-04-26T05:01:47.060 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 19 diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:47.060 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:47.066 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,352321536~6465566,364904528~8203358] 2024-04-26T05:01:47.066 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to 17 diff [50191347~5428687,104551092~5709199,126499545~375794,137230096~4951286,206811004~5690052,235266241~1774805,257797223~2913328,297925974~110671,311160244~70212,337362700~1413194] 2024-04-26T05:01:47.066 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [50191347~5428687,104551092~5709199,126499545~375794,137230096~4951286,206811004~5690052,235266241~1774805,257797223~2913328,297925974~110671,311160244~70212,337362700~1413194] 2024-04-26T05:01:47.072 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [50191347~5428687,104551092~5709199,126499545~375794,137230096~4951286,206811004~5690052,235266241~1774805,257797223~2913328,297925974~110671,311160244~70212,337362700~1413194] 2024-04-26T05:01:47.072 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to 18 diff [50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~4951286,143863691~8561016,206811004~5690052,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311160244~70212,334915611~7171348,355398337~3388765] 2024-04-26T05:01:47.072 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~4951286,143863691~8561016,206811004~5690052,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311160244~70212,334915611~7171348,355398337~3388765] 2024-04-26T05:01:47.078 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~4951286,143863691~8561016,206811004~5690052,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311160244~70212,334915611~7171348,352321536~6465566] 2024-04-26T05:01:47.078 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to 19 diff [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:47.078 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:47.084 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,352321536~6465566,364904528~8203358] 2024-04-26T05:01:47.084 INFO:tasks.workunit.client.0.smithi005.stdout:from 17 to 18 diff [89273018~1622147,98101745~1226828,143863691~8561016,222589835~3430799,270287176~4451997,334915611~7171348,355398337~3388765] 2024-04-26T05:01:47.084 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [89273018~1622147,98101745~1226828,143863691~8561016,222589835~3430799,270287176~4451997,334915611~7171348,355398337~3388765] 2024-04-26T05:01:47.090 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [89273018~1622147,98101745~1226828,143863691~8561016,222589835~3430799,270287176~4451997,334915611~7171348,352321536~6465566] 2024-04-26T05:01:47.090 INFO:tasks.workunit.client.0.smithi005.stdout:from 17 to 19 diff [4666460~3132404,25041300~7162615,47571455~1002103,89273018~1622147,98101745~1226828,139295633~13499362,166860726~2585567,215902769~5180291,222589835~3430799,270287176~4451997,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:47.090 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,25041300~7162615,47571455~1002103,89273018~1622147,98101745~1226828,139295633~13499362,166860726~2585567,215902769~5180291,222589835~3430799,270287176~4451997,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:47.095 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,25041300~7162615,47571455~1002103,89273018~1622147,98101745~1226828,139295633~13499362,166860726~2585567,215902769~5180291,222589835~3430799,270287176~4451997,311000057~4929630,334915611~7171348,352321536~6465566,364904528~8203358] 2024-04-26T05:01:47.095 INFO:tasks.workunit.client.0.smithi005.stdout:from 18 to 19 diff [4666460~3132404,25041300~7162615,47571455~1002103,139295633~5658608,147983914~4811081,166860726~2585567,215902769~5180291,311000057~4929630,364904528~8203358] 2024-04-26T05:01:47.096 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,25041300~7162615,47571455~1002103,139295633~5658608,147983914~4811081,166860726~2585567,215902769~5180291,311000057~4929630,364904528~8203358] 2024-04-26T05:01:47.101 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,25041300~7162615,47571455~1002103,139295633~5658608,147983914~4811081,166860726~2585567,215902769~5180291,311000057~4929630,364904528~8203358] 2024-04-26T05:01:47.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:46 smithi005 ceph-mon[91801]: pgmap v994: 105 pgs: 105 active+clean; 730 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 230 KiB/s rd, 28 MiB/s wr, 348 op/s 2024-04-26T05:01:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:46 smithi005 ceph-mon[91801]: osdmap e873: 8 total, 8 up, 8 in 2024-04-26T05:01:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:46 smithi005 ceph-mon[95809]: pgmap v994: 105 pgs: 105 active+clean; 730 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 230 KiB/s rd, 28 MiB/s wr, 348 op/s 2024-04-26T05:01:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:46 smithi005 ceph-mon[95809]: osdmap e873: 8 total, 8 up, 8 in 2024-04-26T05:01:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:46 smithi111 ceph-mon[86360]: pgmap v994: 105 pgs: 105 active+clean; 730 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 230 KiB/s rd, 28 MiB/s wr, 348 op/s 2024-04-26T05:01:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:46 smithi111 ceph-mon[86360]: osdmap e873: 8 total, 8 up, 8 in 2024-04-26T05:01:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:47.918 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:47.918 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:47.938 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:47.938 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:47.938 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:47.945 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:47.945 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.945 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.951 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:47.951 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.952 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.958 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:47.958 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.958 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.965 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:47.965 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.965 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.971 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:47.971 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.971 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:47.977 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:47.977 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:47.978 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:47.984 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:47.984 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:47.984 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:47.990 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:47.990 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:47.990 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:47.996 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:47.996 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:47.996 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.003 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.003 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.003 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.008 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.009 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.009 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.015 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:48.015 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:48.015 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:48.021 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~12706788] 2024-04-26T05:01:48.021 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:48.021 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:48.027 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,352321536~6465566,363806101~9301785] 2024-04-26T05:01:48.027 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to HEAD diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:48.027 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:48.033 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,352321536~6465566,364904528~8203358] 2024-04-26T05:01:48.033 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to HEAD diff [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:48.033 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:48.039 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,352321536~6465566,364904528~8203358] 2024-04-26T05:01:48.039 INFO:tasks.workunit.client.0.smithi005.stdout:from 17 to HEAD diff [4666460~3132404,25041300~7162615,47571455~1002103,89273018~1622147,98101745~1226828,139295633~13499362,166860726~2585567,215902769~5180291,222589835~3430799,270287176~4451997,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:48.039 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,25041300~7162615,47571455~1002103,89273018~1622147,98101745~1226828,139295633~13499362,166860726~2585567,215902769~5180291,222589835~3430799,270287176~4451997,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:48.044 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,25041300~7162615,47571455~1002103,89273018~1622147,98101745~1226828,139295633~13499362,166860726~2585567,215902769~5180291,222589835~3430799,270287176~4451997,311000057~4929630,334915611~7171348,352321536~6465566,364904528~8203358] 2024-04-26T05:01:48.933 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:48.933 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:48.946 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:48.946 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:48.947 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:48.953 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:48.953 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.953 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.960 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:48.960 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.960 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.967 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:48.967 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.968 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.974 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:48.974 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.974 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.980 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:48.980 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.980 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:48.987 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:48.987 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:48.987 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:48.993 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:48.993 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:48.993 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:48.999 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:48.999 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:48.999 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:49.005 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:49.006 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.006 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.011 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.012 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.012 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.018 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.018 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.018 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.024 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:49.024 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:49.024 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:49.030 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~12706788] 2024-04-26T05:01:49.030 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:49.030 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:49.036 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,352321536~6465566,363806101~9301785] 2024-04-26T05:01:49.036 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to HEAD diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:49.036 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:49.042 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,352321536~6465566,364904528~8203358] 2024-04-26T05:01:49.042 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to HEAD diff [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:49.042 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,355398337~3388765,364904528~8203358] 2024-04-26T05:01:49.048 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,25041300~7162615,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~5709199,126499545~375794,137230096~15564899,166860726~2585567,206811004~5690052,215902769~5180291,222589835~3430799,235266241~1774805,257797223~2913328,270287176~4451997,297925974~110671,311000057~4929630,334915611~7171348,352321536~6465566,364904528~8203358] 2024-04-26T05:01:49.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:48 smithi005 ceph-mon[95809]: pgmap v996: 105 pgs: 105 active+clean; 821 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.8 KiB/s rd, 20 MiB/s wr, 58 op/s 2024-04-26T05:01:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:48 smithi005 ceph-mon[95809]: osdmap e874: 8 total, 8 up, 8 in 2024-04-26T05:01:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:48 smithi005 ceph-mon[91801]: pgmap v996: 105 pgs: 105 active+clean; 821 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.8 KiB/s rd, 20 MiB/s wr, 58 op/s 2024-04-26T05:01:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:48 smithi005 ceph-mon[91801]: osdmap e874: 8 total, 8 up, 8 in 2024-04-26T05:01:49.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:48 smithi111 ceph-mon[86360]: pgmap v996: 105 pgs: 105 active+clean; 821 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.8 KiB/s rd, 20 MiB/s wr, 58 op/s 2024-04-26T05:01:49.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:48 smithi111 ceph-mon[86360]: osdmap e874: 8 total, 8 up, 8 in 2024-04-26T05:01:49.936 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:49.936 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:49.955 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:49.955 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:49.955 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:49.961 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:49.961 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.961 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.968 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:49.968 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.968 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.975 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:49.975 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.975 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.981 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:49.982 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.982 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.988 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:49.988 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.988 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:49.995 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:49.995 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:49.995 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:50.001 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:50.001 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:50.001 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:50.007 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:50.007 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:50.007 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:50.013 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:50.013 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.013 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.019 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.019 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.020 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.025 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.026 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.026 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.032 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:50.032 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:50.032 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:50.038 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~12706788] 2024-04-26T05:01:50.038 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:50.038 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:50.044 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,352321536~6465566,363806101~9301785] 2024-04-26T05:01:50.044 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to HEAD diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:50.044 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,355398337~3388765,364904528~8203358] 2024-04-26T05:01:50.049 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,89273018~1622147,98101745~1226828,104551092~11635364,126499545~375794,137230096~15564899,166860726~2585567,180999320~6640455,197058233~8020163,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,307050062~8879625,325766628~2902956,332067187~10019772,352321536~6465566,364904528~8203358] 2024-04-26T05:01:50.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:49 smithi005 ceph-mon[95809]: osdmap e875: 8 total, 8 up, 8 in 2024-04-26T05:01:50.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:49 smithi005 ceph-mon[91801]: osdmap e875: 8 total, 8 up, 8 in 2024-04-26T05:01:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:49 smithi111 ceph-mon[86360]: osdmap e875: 8 total, 8 up, 8 in 2024-04-26T05:01:50.933 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:50.933 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:50.960 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:50.960 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:50.960 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:50.966 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:50.966 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.966 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.973 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:50.973 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.973 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.979 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:50.979 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.979 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.986 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:50.986 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.986 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.992 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:50.992 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.993 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:50.999 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:50.999 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:50.999 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:51.005 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:51.005 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:51.005 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:51.011 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:51.012 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:51.012 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:51.017 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:51.018 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.018 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.024 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.024 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.024 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.030 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.030 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.030 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.036 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:51.036 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:51.036 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:51.042 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~12706788] 2024-04-26T05:01:51.043 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:51.043 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,355398337~3388765,363806101~9301785] 2024-04-26T05:01:51.048 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,85238503~5656662,98101745~1226828,104551092~11635364,126499545~375794,127854997~7118382,137230096~15564899,166860726~2585567,180999320~6640455,190002781~15075615,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5631968,257797223~2913328,268485267~6253906,297925974~110671,299789837~16139850,325766628~2902956,332067187~10019772,352321536~6465566,363806101~9301785] 2024-04-26T05:01:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:50 smithi005 ceph-mon[91801]: pgmap v999: 105 pgs: 105 active+clean; 866 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 193 KiB/s rd, 31 MiB/s wr, 2.37k op/s 2024-04-26T05:01:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:50 smithi005 ceph-mon[91801]: osdmap e876: 8 total, 8 up, 8 in 2024-04-26T05:01:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:50 smithi005 ceph-mon[95809]: pgmap v999: 105 pgs: 105 active+clean; 866 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 193 KiB/s rd, 31 MiB/s wr, 2.37k op/s 2024-04-26T05:01:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:50 smithi005 ceph-mon[95809]: osdmap e876: 8 total, 8 up, 8 in 2024-04-26T05:01:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:50 smithi111 ceph-mon[86360]: pgmap v999: 105 pgs: 105 active+clean; 866 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 193 KiB/s rd, 31 MiB/s wr, 2.37k op/s 2024-04-26T05:01:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:50 smithi111 ceph-mon[86360]: osdmap e876: 8 total, 8 up, 8 in 2024-04-26T05:01:51.957 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:51.957 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:51.973 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:51.973 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:51.973 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:51.979 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:51.979 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.979 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.986 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:51.987 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.987 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.992 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:51.992 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.992 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.998 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:51.998 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:51.999 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.005 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:52.005 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.005 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.011 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:52.011 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:52.011 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:52.017 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:52.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:52.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:52.023 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:52.024 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:52.024 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:52.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:52.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.041 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.041 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.041 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.047 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:52.047 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:52.047 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~12706788] 2024-04-26T05:01:52.053 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4666460~3132404,12820204~6774326,24844439~7388720,38082429~7946871,47571455~1002103,50191347~5428687,73504814~1175543,85238503~5656662,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~12706788] 2024-04-26T05:01:52.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:51 smithi005 ceph-mon[91801]: osdmap e877: 8 total, 8 up, 8 in 2024-04-26T05:01:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:51 smithi005 ceph-mon[95809]: osdmap e877: 8 total, 8 up, 8 in 2024-04-26T05:01:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:51 smithi111 ceph-mon[86360]: osdmap e877: 8 total, 8 up, 8 in 2024-04-26T05:01:52.963 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:52.963 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:52.979 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:52.979 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:52.979 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:52.985 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:52.985 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.985 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.991 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:52.991 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.991 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.997 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:52.997 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:52.998 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:53.004 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:53.004 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:53.004 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:53.010 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:53.010 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:53.010 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:53.016 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:53.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:53.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:53.023 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:53.023 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:53.023 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:53.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:53.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:53.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:53.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:53.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.040 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.041 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.041 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.046 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.046 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.046 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.052 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,82879792~8015373,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,222589835~3430799,232621452~5634034,257797223~2913328,268485267~6253906,290301230~6519424,297925974~110671,299789837~16139850,325766628~2902956,332067187~11033496,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:53.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:52 smithi005 ceph-mon[95809]: pgmap v1002: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 227 KiB/s rd, 12 MiB/s wr, 5.36k op/s 2024-04-26T05:01:53.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:52 smithi005 ceph-mon[95809]: osdmap e878: 8 total, 8 up, 8 in 2024-04-26T05:01:53.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:52 smithi005 ceph-mon[91801]: pgmap v1002: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 227 KiB/s rd, 12 MiB/s wr, 5.36k op/s 2024-04-26T05:01:53.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:52 smithi005 ceph-mon[91801]: osdmap e878: 8 total, 8 up, 8 in 2024-04-26T05:01:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:52 smithi111 ceph-mon[86360]: pgmap v1002: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 227 KiB/s rd, 12 MiB/s wr, 5.36k op/s 2024-04-26T05:01:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:52 smithi111 ceph-mon[86360]: osdmap e878: 8 total, 8 up, 8 in 2024-04-26T05:01:53.461 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:01:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:01:53] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.43.0" 2024-04-26T05:01:53.966 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:53.966 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:53.986 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:53.987 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:53.987 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:53.993 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:53.993 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:53.993 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.000 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:54.000 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.000 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.006 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:54.006 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.006 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.012 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:54.012 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.012 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.019 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:54.019 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.019 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.025 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:54.025 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:54.025 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:54.031 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:54.031 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:54.031 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:54.037 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:54.038 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:54.038 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:54.043 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:54.043 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:54.043 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:54.050 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:54.050 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:54.050 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:54.055 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,38082429~7946871,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~5690052,215902769~5180291,221093468~7608031,232621452~5634034,246080080~3588531,257797223~2913328,264379302~700168,268485267~6253906,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:54.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:53 smithi005 ceph-mon[91801]: osdmap e879: 8 total, 8 up, 8 in 2024-04-26T05:01:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:53 smithi005 ceph-mon[95809]: osdmap e879: 8 total, 8 up, 8 in 2024-04-26T05:01:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:53 smithi111 ceph-mon[86360]: osdmap e879: 8 total, 8 up, 8 in 2024-04-26T05:01:54.962 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:54.962 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:54.990 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:54.990 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:54.990 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:54.996 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:54.996 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:54.996 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.003 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:55.003 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.003 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.009 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:55.009 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.009 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.015 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:55.016 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.016 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.022 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:55.022 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.022 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:55.028 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:55.028 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:55.028 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:55.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:55.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:55.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:55.041 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:55.041 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:55.041 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:55.047 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:55.047 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:55.047 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,355398337~3388765,360401098~16536112,384421620~5746065] 2024-04-26T05:01:55.052 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,24844439~7388720,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~5634034,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,325766628~17334055,352321536~6465566,360401098~16536112,384421620~5746065] 2024-04-26T05:01:55.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:54 smithi005 ceph-mon[91801]: pgmap v1005: 105 pgs: 105 active+clean; 844 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 161 KiB/s rd, 6.5 KiB/s wr, 3.97k op/s 2024-04-26T05:01:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:54 smithi005 ceph-mon[91801]: osdmap e880: 8 total, 8 up, 8 in 2024-04-26T05:01:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:54 smithi005 ceph-mon[95809]: pgmap v1005: 105 pgs: 105 active+clean; 844 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 161 KiB/s rd, 6.5 KiB/s wr, 3.97k op/s 2024-04-26T05:01:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:54 smithi005 ceph-mon[95809]: osdmap e880: 8 total, 8 up, 8 in 2024-04-26T05:01:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:54 smithi111 ceph-mon[86360]: pgmap v1005: 105 pgs: 105 active+clean; 844 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 161 KiB/s rd, 6.5 KiB/s wr, 3.97k op/s 2024-04-26T05:01:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:54 smithi111 ceph-mon[86360]: osdmap e880: 8 total, 8 up, 8 in 2024-04-26T05:01:55.975 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:55.975 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:56.002 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:56.003 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:56.003 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:56.010 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:56.010 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.010 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.016 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:56.016 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.016 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.022 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:56.022 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.022 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.028 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:56.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:56.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:56.041 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:56.041 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:56.041 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:56.047 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:56.047 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:56.047 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:56.053 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:56.053 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:56.053 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,355398337~3388765,359690727~17246483,383510826~6665543] 2024-04-26T05:01:56.059 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,12820204~6774326,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,63000434~6755362,73504814~1175543,76265946~14629219,98101745~1226828,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166860726~2585567,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,325766628~17334055,352321536~24615674,383510826~6665543] 2024-04-26T05:01:56.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:55 smithi005 ceph-mon[91801]: osdmap e881: 8 total, 8 up, 8 in 2024-04-26T05:01:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:55 smithi005 ceph-mon[95809]: osdmap e881: 8 total, 8 up, 8 in 2024-04-26T05:01:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:55 smithi111 ceph-mon[86360]: osdmap e881: 8 total, 8 up, 8 in 2024-04-26T05:01:56.986 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:56.986 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.009 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:57.009 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.009 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.016 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:57.016 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.016 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.023 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.023 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.023 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.042 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.042 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.042 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.048 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.048 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:57.048 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:57.054 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:57.054 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:57.054 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214] 2024-04-26T05:01:57.061 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [814559~6984305,10138532~11147996,22694932~9538227,33990443~728587,34972837~11056463,47571455~1002103,50191347~5428687,56710724~189312,60560864~9194932,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,166755333~5292336,180999320~6640455,188005676~17072720,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705] 2024-04-26T05:01:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:56 smithi005 ceph-mon[91801]: pgmap v1008: 105 pgs: 105 active+clean; 844 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 141 KiB/s rd, 6.2 KiB/s wr, 1.11k op/s 2024-04-26T05:01:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:56 smithi005 ceph-mon[91801]: osdmap e882: 8 total, 8 up, 8 in 2024-04-26T05:01:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:56 smithi005 ceph-mon[95809]: pgmap v1008: 105 pgs: 105 active+clean; 844 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 141 KiB/s rd, 6.2 KiB/s wr, 1.11k op/s 2024-04-26T05:01:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:56 smithi005 ceph-mon[95809]: osdmap e882: 8 total, 8 up, 8 in 2024-04-26T05:01:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:56 smithi111 ceph-mon[86360]: pgmap v1008: 105 pgs: 105 active+clean; 844 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 141 KiB/s rd, 6.2 KiB/s wr, 1.11k op/s 2024-04-26T05:01:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:56 smithi111 ceph-mon[86360]: osdmap e882: 8 total, 8 up, 8 in 2024-04-26T05:01:57.781 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.781 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.804 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:57.804 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.804 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:57.811 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:57.811 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.811 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.817 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.818 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.818 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.824 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.824 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.824 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.830 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.830 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.830 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.836 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.836 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.836 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:57.842 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:57.842 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:57.843 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,355398337~3388765,359690727~17246483,382845155~7331214,397306655~47324] 2024-04-26T05:01:57.848 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,126499545~375794,127854997~7118382,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~6253906,282785062~2730194,288524467~27405220,316749913~2883833,323712297~19388386,352321536~24615674,381681664~8494705,397306655~47324] 2024-04-26T05:01:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:57 smithi005 ceph-mon[91801]: osdmap e883: 8 total, 8 up, 8 in 2024-04-26T05:01:58.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:57 smithi005 ceph-mon[91801]: osdmap e884: 8 total, 8 up, 8 in 2024-04-26T05:01:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:57 smithi005 ceph-mon[95809]: osdmap e883: 8 total, 8 up, 8 in 2024-04-26T05:01:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:57 smithi005 ceph-mon[95809]: osdmap e884: 8 total, 8 up, 8 in 2024-04-26T05:01:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:57 smithi111 ceph-mon[86360]: osdmap e883: 8 total, 8 up, 8 in 2024-04-26T05:01:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:57 smithi111 ceph-mon[86360]: osdmap e884: 8 total, 8 up, 8 in 2024-04-26T05:01:58.775 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:58.775 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:58.806 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:58.806 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:58.806 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:58.813 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:58.813 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.813 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.819 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:58.820 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.820 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.826 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:58.826 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.826 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.832 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:58.832 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.833 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.839 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:58.839 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.839 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~15097210,125863971~9109408,137230096~15564899,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:58.845 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~15153106,125829120~9144259,137230096~15564899,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~6153518,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27405220,316749913~3660721,323712297~19600271,346214060~4698831,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:59.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:58 smithi005 ceph-mon[91801]: pgmap v1011: 105 pgs: 105 active+clean; 808 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 755 op/s 2024-04-26T05:01:59.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:01:58 smithi005 ceph-mon[91801]: osdmap e885: 8 total, 8 up, 8 in 2024-04-26T05:01:59.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:58 smithi005 ceph-mon[95809]: pgmap v1011: 105 pgs: 105 active+clean; 808 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 755 op/s 2024-04-26T05:01:59.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:01:58 smithi005 ceph-mon[95809]: osdmap e885: 8 total, 8 up, 8 in 2024-04-26T05:01:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:58 smithi111 ceph-mon[86360]: pgmap v1011: 105 pgs: 105 active+clean; 808 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 755 op/s 2024-04-26T05:01:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:01:58 smithi111 ceph-mon[86360]: osdmap e885: 8 total, 8 up, 8 in 2024-04-26T05:01:59.779 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:59.779 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:59.806 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:59.806 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:59.806 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:01:59.813 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:01:59.813 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.813 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.820 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:59.820 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.820 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.826 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:59.826 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.826 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.832 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:01:59.832 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.832 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,323712297~27200594,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:01:59.838 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56710724~189312,60560864~9716984,73504814~1175543,76265946~14629219,96669846~4086612,101033350~18334964,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,257797223~2913328,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,323712297~27200594,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:00.782 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:00.782 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:00.807 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:00.807 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:00.807 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:00.816 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:00.816 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:00.816 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:00.822 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:00.823 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:00.823 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:00.829 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:00.829 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:00.829 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~7166730,101089246~18279068,125863971~9109408,137230096~17776342,156533592~1041666,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,256469328~6792010,264379302~700168,268485267~7758251,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:00.835 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,73504814~1175543,76265946~14629219,93866620~25501694,125829120~9144259,137230096~17776342,156533592~1041666,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,215902769~12798730,232621452~7887016,246080080~7521465,255984802~7276536,264379302~700168,268485267~7758251,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:00 smithi111 ceph-mon[86360]: pgmap v1014: 105 pgs: 105 active+clean; 759 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 6.2 KiB/s wr, 1.54k op/s 2024-04-26T05:02:01.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:00 smithi111 ceph-mon[86360]: osdmap e886: 8 total, 8 up, 8 in 2024-04-26T05:02:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:00 smithi005 ceph-mon[91801]: pgmap v1014: 105 pgs: 105 active+clean; 759 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 6.2 KiB/s wr, 1.54k op/s 2024-04-26T05:02:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:00 smithi005 ceph-mon[91801]: osdmap e886: 8 total, 8 up, 8 in 2024-04-26T05:02:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:00 smithi005 ceph-mon[95809]: pgmap v1014: 105 pgs: 105 active+clean; 759 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 6.2 KiB/s wr, 1.54k op/s 2024-04-26T05:02:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:00 smithi005 ceph-mon[95809]: osdmap e886: 8 total, 8 up, 8 in 2024-04-26T05:02:01.792 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:01.792 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:01.815 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:01.815 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:01.815 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:01.821 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:01.822 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:01.822 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:01.827 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:01.828 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:01.828 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~13663295,72793401~18101764,93866620~7166730,101089246~18279068,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,256469328~19774190,282785062~3317443,288524467~27445924,316749913~3660721,320955023~29957868,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:01.833 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~13663295,72755885~18139280,93866620~25501694,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~9541100,255984802~20258716,281018368~5084137,288524467~27445924,316749913~3660721,320955023~29957868,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:02.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:01 smithi005 ceph-mon[91801]: osdmap e887: 8 total, 8 up, 8 in 2024-04-26T05:02:02.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:01 smithi005 ceph-mon[95809]: osdmap e887: 8 total, 8 up, 8 in 2024-04-26T05:02:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:01 smithi111 ceph-mon[86360]: osdmap e887: 8 total, 8 up, 8 in 2024-04-26T05:02:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:02.783 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:02.784 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:02.807 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:02.807 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:02.807 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:02.813 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:02.813 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:02.813 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~23558824,47571455~1002103,50191347~5428687,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~9109408,137230096~21684220,161552797~233185,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~17246483,378654936~2811140,382845155~7331214,396077477~1902872] 2024-04-26T05:02:02.819 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~46029300,47571455~1002103,50191347~5428687,56614553~34280612,93866620~25501694,121634816~2416291,125829120~9144259,137230096~21684220,159383552~2402430,164569697~386756,166755333~5292336,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~24615674,378654936~2811140,381681664~8494705,396077477~1902872] 2024-04-26T05:02:03.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:02 smithi111 ceph-mon[86360]: pgmap v1017: 105 pgs: 105 active+clean; 679 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 6.2 KiB/s wr, 1.28k op/s 2024-04-26T05:02:03.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:02 smithi111 ceph-mon[86360]: osdmap e888: 8 total, 8 up, 8 in 2024-04-26T05:02:03.170 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:02 smithi111 ceph-mon[86360]: osdmap e889: 8 total, 8 up, 8 in 2024-04-26T05:02:03.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:02 smithi005 ceph-mon[95809]: pgmap v1017: 105 pgs: 105 active+clean; 679 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 6.2 KiB/s wr, 1.28k op/s 2024-04-26T05:02:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:02 smithi005 ceph-mon[95809]: osdmap e888: 8 total, 8 up, 8 in 2024-04-26T05:02:03.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:02 smithi005 ceph-mon[95809]: osdmap e889: 8 total, 8 up, 8 in 2024-04-26T05:02:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:02 smithi005 ceph-mon[91801]: pgmap v1017: 105 pgs: 105 active+clean; 679 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 6.2 KiB/s wr, 1.28k op/s 2024-04-26T05:02:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:02 smithi005 ceph-mon[91801]: osdmap e888: 8 total, 8 up, 8 in 2024-04-26T05:02:03.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:02 smithi005 ceph-mon[91801]: osdmap e889: 8 total, 8 up, 8 in 2024-04-26T05:02:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:02:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:02:03] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2024-04-26T05:02:03.803 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:03.803 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:03.826 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:03.826 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:03.826 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56614553~16141332,72793401~18101764,93866620~7166730,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~6640455,188005676~18317484,206811004~7060422,214138291~26370177,244060445~9541100,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:03.832 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56614553~34280612,93866620~25501694,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~6640455,188005676~18317484,206811004~7060422,213909504~26598964,244060445~32183073,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:04.805 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:04.805 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [814559~20471969,22470476~33149558,56441850~16314035,72793401~28239949,101089246~18279068,123388044~663063,125863971~33050345,161552797~233185,164569697~386756,166051702~5995967,180999320~25323840,206811004~7060422,214138291~39463254,255108052~876750,256469328~19774190,282785062~3317443,288524467~27445924,316749913~34162978,355398337~3388765,359690727~21775349,382845155~7331214,394539851~3440498] 2024-04-26T05:02:04.824 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~55620034,56441850~62926464,121634816~2416291,125829120~33085196,159383552~2402430,163577856~8469813,180999320~25323840,206811004~7060422,213909504~62334014,281018368~5084137,288524467~62388424,352321536~29144540,381681664~8494705,394264576~3715773] 2024-04-26T05:02:05.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:04 smithi005 ceph-mon[91801]: pgmap v1020: 105 pgs: 105 active+clean; 607 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 77 KiB/s rd, 6.0 KiB/s wr, 954 op/s 2024-04-26T05:02:05.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:04 smithi005 ceph-mon[91801]: osdmap e890: 8 total, 8 up, 8 in 2024-04-26T05:02:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:04 smithi005 ceph-mon[95809]: pgmap v1020: 105 pgs: 105 active+clean; 607 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 77 KiB/s rd, 6.0 KiB/s wr, 954 op/s 2024-04-26T05:02:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:04 smithi005 ceph-mon[95809]: osdmap e890: 8 total, 8 up, 8 in 2024-04-26T05:02:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:04 smithi111 ceph-mon[86360]: pgmap v1020: 105 pgs: 105 active+clean; 607 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 77 KiB/s rd, 6.0 KiB/s wr, 954 op/s 2024-04-26T05:02:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:04 smithi111 ceph-mon[86360]: osdmap e890: 8 total, 8 up, 8 in 2024-04-26T05:02:05.823 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (39991 ms) 2024-04-26T05:02:05.823 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-04-26T05:02:05.823 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:05 smithi005 ceph-mon[91801]: osdmap e891: 8 total, 8 up, 8 in 2024-04-26T05:02:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:05 smithi005 ceph-mon[95809]: osdmap e891: 8 total, 8 up, 8 in 2024-04-26T05:02:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:05 smithi111 ceph-mon[86360]: osdmap e891: 8 total, 8 up, 8 in 2024-04-26T05:02:06.917 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1094 ms) 2024-04-26T05:02:06.917 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-04-26T05:02:06.917 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:07.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:06 smithi005 ceph-mon[95809]: pgmap v1023: 105 pgs: 105 active+clean; 607 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 76 KiB/s rd, 6.0 KiB/s wr, 452 op/s 2024-04-26T05:02:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:06 smithi005 ceph-mon[95809]: osdmap e892: 8 total, 8 up, 8 in 2024-04-26T05:02:07.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:06 smithi005 ceph-mon[91801]: pgmap v1023: 105 pgs: 105 active+clean; 607 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 76 KiB/s rd, 6.0 KiB/s wr, 452 op/s 2024-04-26T05:02:07.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:06 smithi005 ceph-mon[91801]: osdmap e892: 8 total, 8 up, 8 in 2024-04-26T05:02:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:06 smithi111 ceph-mon[86360]: pgmap v1023: 105 pgs: 105 active+clean; 607 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 76 KiB/s rd, 6.0 KiB/s wr, 452 op/s 2024-04-26T05:02:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:06 smithi111 ceph-mon[86360]: osdmap e892: 8 total, 8 up, 8 in 2024-04-26T05:02:07.899 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [1540256~91903,5575030~76222,6665325~33676,8238394~68617,10946844~45713,17645445~16746,19800751~91331,19910120~44579,20097651~16664,20603928~43845] to clone 2024-04-26T05:02:07.900 INFO:tasks.workunit.client.0.smithi005.stdout: diff was [0~1632159,4194304~4112707,8388608~2603949,16777216~3870557] 2024-04-26T05:02:07.904 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (987 ms) 2024-04-26T05:02:07.904 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-04-26T05:02:07.904 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:07.922 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [783471~69487,2391593~44650,3756677~6363,9146123~74026,12524640~55060,13329931~15713,16398887~77525,17787847~58768,19936980~12856,20573733~87228] 2024-04-26T05:02:07.933 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (29 ms) 2024-04-26T05:02:07.933 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-04-26T05:02:07.933 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:08.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:07 smithi005 ceph-mon[91801]: osdmap e893: 8 total, 8 up, 8 in 2024-04-26T05:02:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:07 smithi005 ceph-mon[91801]: osdmap e894: 8 total, 8 up, 8 in 2024-04-26T05:02:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:07 smithi005 ceph-mon[95809]: osdmap e893: 8 total, 8 up, 8 in 2024-04-26T05:02:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:07 smithi005 ceph-mon[95809]: osdmap e894: 8 total, 8 up, 8 in 2024-04-26T05:02:08.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:07 smithi111 ceph-mon[86360]: osdmap e893: 8 total, 8 up, 8 in 2024-04-26T05:02:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:07 smithi111 ceph-mon[86360]: osdmap e894: 8 total, 8 up, 8 in 2024-04-26T05:02:09.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:08 smithi005 ceph-mon[91801]: pgmap v1026: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 513 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.0 MiB/s wr, 135 op/s 2024-04-26T05:02:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:08 smithi005 ceph-mon[95809]: pgmap v1026: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 513 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.0 MiB/s wr, 135 op/s 2024-04-26T05:02:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:08 smithi111 ceph-mon[86360]: pgmap v1026: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 513 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.0 MiB/s wr, 135 op/s 2024-04-26T05:02:09.923 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1990 ms) 2024-04-26T05:02:09.924 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-04-26T05:02:09.924 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:09.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (53 ms) 2024-04-26T05:02:09.976 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-04-26T05:02:09.976 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:10.061 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (85 ms) 2024-04-26T05:02:10.061 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTryAcquireLock 2024-04-26T05:02:10.061 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:10.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:09 smithi005 ceph-mon[91801]: osdmap e895: 8 total, 8 up, 8 in 2024-04-26T05:02:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:09 smithi005 ceph-mon[95809]: osdmap e895: 8 total, 8 up, 8 in 2024-04-26T05:02:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:09 smithi111 ceph-mon[86360]: osdmap e895: 8 total, 8 up, 8 in 2024-04-26T05:02:11.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:10 smithi005 ceph-mon[91801]: pgmap v1029: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 446 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 6.0 MiB/s wr, 308 op/s 2024-04-26T05:02:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:10 smithi005 ceph-mon[91801]: osdmap e896: 8 total, 8 up, 8 in 2024-04-26T05:02:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:10 smithi005 ceph-mon[95809]: pgmap v1029: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 446 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 6.0 MiB/s wr, 308 op/s 2024-04-26T05:02:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:10 smithi005 ceph-mon[95809]: osdmap e896: 8 total, 8 up, 8 in 2024-04-26T05:02:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:10 smithi111 ceph-mon[86360]: pgmap v1029: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 446 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 6.0 MiB/s wr, 308 op/s 2024-04-26T05:02:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:10 smithi111 ceph-mon[86360]: osdmap e896: 8 total, 8 up, 8 in 2024-04-26T05:02:13.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:12 smithi005 ceph-mon[91801]: pgmap v1031: 105 pgs: 105 active+clean; 505 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 9.7 MiB/s wr, 267 op/s 2024-04-26T05:02:13.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:12 smithi005 ceph-mon[91801]: osdmap e897: 8 total, 8 up, 8 in 2024-04-26T05:02:13.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:12 smithi005 ceph-mon[95809]: pgmap v1031: 105 pgs: 105 active+clean; 505 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 9.7 MiB/s wr, 267 op/s 2024-04-26T05:02:13.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:12 smithi005 ceph-mon[95809]: osdmap e897: 8 total, 8 up, 8 in 2024-04-26T05:02:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:12 smithi111 ceph-mon[86360]: pgmap v1031: 105 pgs: 105 active+clean; 505 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 9.7 MiB/s wr, 267 op/s 2024-04-26T05:02:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:12 smithi111 ceph-mon[86360]: osdmap e897: 8 total, 8 up, 8 in 2024-04-26T05:02:13.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:02:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:02:13] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2024-04-26T05:02:15.096 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateTryAcquireLock (5035 ms) 2024-04-26T05:02:15.096 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-04-26T05:02:15.096 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T05:02:15.096 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-04-26T05:02:15.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:14 smithi005 ceph-mon[91801]: pgmap v1033: 105 pgs: 105 active+clean; 548 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 9.2 MiB/s wr, 248 op/s 2024-04-26T05:02:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:14 smithi005 ceph-mon[95809]: pgmap v1033: 105 pgs: 105 active+clean; 548 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 9.2 MiB/s wr, 248 op/s 2024-04-26T05:02:15.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:14 smithi111 ceph-mon[86360]: pgmap v1033: 105 pgs: 105 active+clean; 548 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 9.2 MiB/s wr, 248 op/s 2024-04-26T05:02:15.840 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 15 tests from DiffIterateTest/0 (69272 ms total) 2024-04-26T05:02:15.840 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T05:02:15.840 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 15 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-04-26T05:02:15.880 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-04-26T05:02:16.157 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:15 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:16.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:15 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:16.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:15 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:16 smithi005 ceph-mon[91801]: pgmap v1034: 105 pgs: 105 active+clean; 548 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.8 MiB/s wr, 212 op/s 2024-04-26T05:02:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:16 smithi005 ceph-mon[91801]: osdmap e898: 8 total, 8 up, 8 in 2024-04-26T05:02:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:16 smithi005 ceph-mon[95809]: pgmap v1034: 105 pgs: 105 active+clean; 548 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.8 MiB/s wr, 212 op/s 2024-04-26T05:02:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:16 smithi005 ceph-mon[95809]: osdmap e898: 8 total, 8 up, 8 in 2024-04-26T05:02:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:16 smithi111 ceph-mon[86360]: pgmap v1034: 105 pgs: 105 active+clean; 548 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.8 MiB/s wr, 212 op/s 2024-04-26T05:02:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:16 smithi111 ceph-mon[86360]: osdmap e898: 8 total, 8 up, 8 in 2024-04-26T05:02:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:17.861 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:17 smithi111 ceph-mon[86360]: osdmap e899: 8 total, 8 up, 8 in 2024-04-26T05:02:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:17 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2766160279' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:02:18.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:17 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:02:18.116 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:17 smithi005 ceph-mon[95809]: osdmap e899: 8 total, 8 up, 8 in 2024-04-26T05:02:18.116 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:17 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2766160279' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:02:18.116 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:17 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:02:18.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:17 smithi005 ceph-mon[91801]: osdmap e899: 8 total, 8 up, 8 in 2024-04-26T05:02:18.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:17 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2766160279' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:02:18.117 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:17 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T05:02:18.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[91801]: pgmap v1037: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[91801]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[91801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[91801]: osdmap e900: 8 total, 8 up, 8 in 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[95809]: pgmap v1037: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[95809]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[95809]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T05:02:18.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:18 smithi005 ceph-mon[95809]: osdmap e900: 8 total, 8 up, 8 in 2024-04-26T05:02:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:18 smithi111 ceph-mon[86360]: pgmap v1037: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:18 smithi111 ceph-mon[86360]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T05:02:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:18 smithi111 ceph-mon[86360]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi005-77850-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T05:02:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:18 smithi111 ceph-mon[86360]: osdmap e900: 8 total, 8 up, 8 in 2024-04-26T05:02:19.876 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [2749356~17732,3906590~84474,10951712~76321,12109856~54911,13342927~4685,17805686~66442,18168176~29432,19302900~18763,19967733~101733,20088942~40915] 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: osdmap e901: 8 total, 8 up, 8 in 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: osdmap e901: 8 total, 8 up, 8 in 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:20.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:19 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: osdmap e901: 8 total, 8 up, 8 in 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:19 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:20.877 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [0~20971520] 2024-04-26T05:02:20.886 INFO:tasks.workunit.client.0.smithi005.stdout: diff was [0~20971520] 2024-04-26T05:02:20.890 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5010 ms) 2024-04-26T05:02:20.890 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministic 2024-04-26T05:02:20.891 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[91801]: pgmap v1040: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[91801]: pgmap v1041: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[91801]: pgmap v1042: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[91801]: osdmap e902: 8 total, 8 up, 8 in 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[91801]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[95809]: pgmap v1040: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[95809]: pgmap v1041: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[95809]: pgmap v1042: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[95809]: osdmap e902: 8 total, 8 up, 8 in 2024-04-26T05:02:21.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:20 smithi005 ceph-mon[95809]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T05:02:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:20 smithi111 ceph-mon[86360]: pgmap v1040: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:20 smithi111 ceph-mon[86360]: pgmap v1041: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:20 smithi111 ceph-mon[86360]: pgmap v1042: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:20 smithi111 ceph-mon[86360]: osdmap e902: 8 total, 8 up, 8 in 2024-04-26T05:02:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:20 smithi111 ceph-mon[86360]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T05:02:22.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:21 smithi005 ceph-mon[91801]: osdmap e903: 8 total, 8 up, 8 in 2024-04-26T05:02:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:21 smithi005 ceph-mon[95809]: osdmap e903: 8 total, 8 up, 8 in 2024-04-26T05:02:22.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:21 smithi111 ceph-mon[86360]: osdmap e903: 8 total, 8 up, 8 in 2024-04-26T05:02:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:22 smithi005 ceph-mon[91801]: pgmap v1045: 105 pgs: 19 creating+peering, 86 active+clean; 6.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 140 KiB/s wr, 23 op/s 2024-04-26T05:02:23.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:22 smithi005 ceph-mon[91801]: osdmap e904: 8 total, 8 up, 8 in 2024-04-26T05:02:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:22 smithi005 ceph-mon[95809]: pgmap v1045: 105 pgs: 19 creating+peering, 86 active+clean; 6.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 140 KiB/s wr, 23 op/s 2024-04-26T05:02:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:22 smithi005 ceph-mon[95809]: osdmap e904: 8 total, 8 up, 8 in 2024-04-26T05:02:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:22 smithi111 ceph-mon[86360]: pgmap v1045: 105 pgs: 19 creating+peering, 86 active+clean; 6.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 140 KiB/s wr, 23 op/s 2024-04-26T05:02:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:22 smithi111 ceph-mon[86360]: osdmap e904: 8 total, 8 up, 8 in 2024-04-26T05:02:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:02:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:02:23] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2024-04-26T05:02:23.909 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministic (3018 ms) 2024-04-26T05:02:23.909 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicPP 2024-04-26T05:02:23.909 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:24.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:23 smithi005 ceph-mon[91801]: osdmap e905: 8 total, 8 up, 8 in 2024-04-26T05:02:24.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:23 smithi005 ceph-mon[95809]: osdmap e905: 8 total, 8 up, 8 in 2024-04-26T05:02:24.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:23 smithi111 ceph-mon[86360]: osdmap e905: 8 total, 8 up, 8 in 2024-04-26T05:02:25.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:24 smithi005 ceph-mon[91801]: pgmap v1048: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 276 KiB/s wr, 68 op/s 2024-04-26T05:02:25.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:24 smithi005 ceph-mon[91801]: osdmap e906: 8 total, 8 up, 8 in 2024-04-26T05:02:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:24 smithi005 ceph-mon[95809]: pgmap v1048: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 276 KiB/s wr, 68 op/s 2024-04-26T05:02:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:24 smithi005 ceph-mon[95809]: osdmap e906: 8 total, 8 up, 8 in 2024-04-26T05:02:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:24 smithi111 ceph-mon[86360]: pgmap v1048: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 276 KiB/s wr, 68 op/s 2024-04-26T05:02:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:24 smithi111 ceph-mon[86360]: osdmap e906: 8 total, 8 up, 8 in 2024-04-26T05:02:26.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:25 smithi005 ceph-mon[91801]: osdmap e907: 8 total, 8 up, 8 in 2024-04-26T05:02:26.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:25 smithi005 ceph-mon[95809]: osdmap e907: 8 total, 8 up, 8 in 2024-04-26T05:02:26.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:25 smithi111 ceph-mon[86360]: osdmap e907: 8 total, 8 up, 8 in 2024-04-26T05:02:26.931 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicPP (3023 ms) 2024-04-26T05:02:26.932 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-04-26T05:02:26.932 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:27.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:26 smithi005 ceph-mon[91801]: pgmap v1051: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 174 KiB/s wr, 51 op/s 2024-04-26T05:02:27.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:26 smithi005 ceph-mon[91801]: osdmap e908: 8 total, 8 up, 8 in 2024-04-26T05:02:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:26 smithi005 ceph-mon[95809]: pgmap v1051: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 174 KiB/s wr, 51 op/s 2024-04-26T05:02:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:26 smithi005 ceph-mon[95809]: osdmap e908: 8 total, 8 up, 8 in 2024-04-26T05:02:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:26 smithi111 ceph-mon[86360]: pgmap v1051: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 174 KiB/s wr, 51 op/s 2024-04-26T05:02:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:26 smithi111 ceph-mon[86360]: osdmap e908: 8 total, 8 up, 8 in 2024-04-26T05:02:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:27 smithi005 ceph-mon[91801]: osdmap e909: 8 total, 8 up, 8 in 2024-04-26T05:02:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:27 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:28.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:27 smithi005 ceph-mon[91801]: osdmap e910: 8 total, 8 up, 8 in 2024-04-26T05:02:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:27 smithi005 ceph-mon[95809]: osdmap e909: 8 total, 8 up, 8 in 2024-04-26T05:02:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:27 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:27 smithi005 ceph-mon[95809]: osdmap e910: 8 total, 8 up, 8 in 2024-04-26T05:02:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:27 smithi111 ceph-mon[86360]: osdmap e909: 8 total, 8 up, 8 in 2024-04-26T05:02:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:27 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:02:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:27 smithi111 ceph-mon[86360]: osdmap e910: 8 total, 8 up, 8 in 2024-04-26T05:02:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:28 smithi005 ceph-mon[95809]: pgmap v1054: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 12 KiB/s wr, 114 op/s 2024-04-26T05:02:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:28 smithi005 ceph-mon[95809]: osdmap e911: 8 total, 8 up, 8 in 2024-04-26T05:02:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:28 smithi005 ceph-mon[91801]: pgmap v1054: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 12 KiB/s wr, 114 op/s 2024-04-26T05:02:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:28 smithi005 ceph-mon[91801]: osdmap e911: 8 total, 8 up, 8 in 2024-04-26T05:02:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:28 smithi111 ceph-mon[86360]: pgmap v1054: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 12 KiB/s wr, 114 op/s 2024-04-26T05:02:29.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:28 smithi111 ceph-mon[86360]: osdmap e911: 8 total, 8 up, 8 in 2024-04-26T05:02:29.804 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (2873 ms) 2024-04-26T05:02:29.805 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTruncate 2024-04-26T05:02:29.805 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:30 smithi005 ceph-mon[95809]: pgmap v1057: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 12 KiB/s wr, 114 op/s 2024-04-26T05:02:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:30 smithi005 ceph-mon[95809]: osdmap e912: 8 total, 8 up, 8 in 2024-04-26T05:02:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:30 smithi005 ceph-mon[91801]: pgmap v1057: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 12 KiB/s wr, 114 op/s 2024-04-26T05:02:31.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:30 smithi005 ceph-mon[91801]: osdmap e912: 8 total, 8 up, 8 in 2024-04-26T05:02:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:30 smithi111 ceph-mon[86360]: pgmap v1057: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 12 KiB/s wr, 114 op/s 2024-04-26T05:02:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:30 smithi111 ceph-mon[86360]: osdmap e912: 8 total, 8 up, 8 in 2024-04-26T05:02:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:31 smithi005 ceph-mon[91801]: osdmap e913: 8 total, 8 up, 8 in 2024-04-26T05:02:32.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:31 smithi005 ceph-mon[95809]: osdmap e913: 8 total, 8 up, 8 in 2024-04-26T05:02:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:31 smithi111 ceph-mon[86360]: osdmap e913: 8 total, 8 up, 8 in 2024-04-26T05:02:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:32.839 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateTruncate (3035 ms) 2024-04-26T05:02:32.839 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateWriteAndTruncate 2024-04-26T05:02:32.839 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:33.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:32 smithi111 ceph-mon[86360]: pgmap v1060: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 132 KiB/s wr, 60 op/s 2024-04-26T05:02:33.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:32 smithi111 ceph-mon[86360]: osdmap e914: 8 total, 8 up, 8 in 2024-04-26T05:02:33.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:32 smithi005 ceph-mon[91801]: pgmap v1060: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 132 KiB/s wr, 60 op/s 2024-04-26T05:02:33.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:32 smithi005 ceph-mon[91801]: osdmap e914: 8 total, 8 up, 8 in 2024-04-26T05:02:33.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:32 smithi005 ceph-mon[95809]: pgmap v1060: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 132 KiB/s wr, 60 op/s 2024-04-26T05:02:33.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:32 smithi005 ceph-mon[95809]: osdmap e914: 8 total, 8 up, 8 in 2024-04-26T05:02:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:02:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:02:33] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.43.0" 2024-04-26T05:02:34.185 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:33 smithi111 ceph-mon[86360]: osdmap e915: 8 total, 8 up, 8 in 2024-04-26T05:02:34.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:33 smithi005 ceph-mon[91801]: osdmap e915: 8 total, 8 up, 8 in 2024-04-26T05:02:34.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:33 smithi005 ceph-mon[95809]: osdmap e915: 8 total, 8 up, 8 in 2024-04-26T05:02:35.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:34 smithi005 ceph-mon[91801]: pgmap v1063: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 136 KiB/s wr, 83 op/s 2024-04-26T05:02:35.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:34 smithi005 ceph-mon[91801]: osdmap e916: 8 total, 8 up, 8 in 2024-04-26T05:02:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:34 smithi005 ceph-mon[95809]: pgmap v1063: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 136 KiB/s wr, 83 op/s 2024-04-26T05:02:35.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:34 smithi005 ceph-mon[95809]: osdmap e916: 8 total, 8 up, 8 in 2024-04-26T05:02:35.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:34 smithi111 ceph-mon[86360]: pgmap v1063: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 136 KiB/s wr, 83 op/s 2024-04-26T05:02:35.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:34 smithi111 ceph-mon[86360]: osdmap e916: 8 total, 8 up, 8 in 2024-04-26T05:02:35.854 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateWriteAndTruncate (3015 ms) 2024-04-26T05:02:35.854 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-04-26T05:02:35.854 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:02:35.926 INFO:tasks.workunit.client.0.smithi005.stdout: i=0 exists [3907985~6761365,31455219~2988311,67237627~7300690,81869044~2405596,261421777~3682745,280096820~3519235,290699362~3035355,313132291~1239533,336626861~5414605,391557501~5719541] wrote [3907985~6761365,31455219~2988311,67237627~7300690,81869044~2405596,261421777~3682745,280096820~3519235,290699362~3035355,313132291~1239533,336626861~5414605,391557501~5719541] 2024-04-26T05:02:36.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:35 smithi005 ceph-mon[91801]: osdmap e917: 8 total, 8 up, 8 in 2024-04-26T05:02:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:35 smithi005 ceph-mon[95809]: osdmap e917: 8 total, 8 up, 8 in 2024-04-26T05:02:36.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:35 smithi111 ceph-mon[86360]: osdmap e917: 8 total, 8 up, 8 in 2024-04-26T05:02:36.873 INFO:tasks.workunit.client.0.smithi005.stdout: i=1 exists [2971003~7698347,31455219~2988311,38349730~1128998,51182602~7912451,67237627~7300690,81869044~2405596,103478426~1352441,180492905~1175234,226302548~7648367,261421777~3682745,272991490~2616824,280096820~3519235,290699362~3035355,313132291~1239533,336626861~6999065,367225159~6878947,391557501~5719541] wrote [2971003~1202267,38349730~1128998,51182602~7912451,103478426~1352441,180492905~1175234,226302548~7648367,272991490~2616824,339957064~3668862,367225159~6878947] 2024-04-26T05:02:37.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:36 smithi005 ceph-mon[91801]: pgmap v1066: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 KiB/s wr, 22 op/s 2024-04-26T05:02:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:36 smithi005 ceph-mon[91801]: osdmap e918: 8 total, 8 up, 8 in 2024-04-26T05:02:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:36 smithi005 ceph-mon[95809]: pgmap v1066: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 KiB/s wr, 22 op/s 2024-04-26T05:02:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:36 smithi005 ceph-mon[95809]: osdmap e918: 8 total, 8 up, 8 in 2024-04-26T05:02:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:36 smithi111 ceph-mon[86360]: pgmap v1066: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 KiB/s wr, 22 op/s 2024-04-26T05:02:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:36 smithi111 ceph-mon[86360]: osdmap e918: 8 total, 8 up, 8 in 2024-04-26T05:02:37.883 INFO:tasks.workunit.client.0.smithi005.stdout: i=2 exists [2971003~7698347,11489330~2900752,28855924~6418790,38349730~1128998,51182602~7912451,67237627~7300690,81869044~2405596,103478426~1352441,159248657~3912237,180492905~1175234,215842100~4042443,226302548~7648367,244358165~5723321,251380213~412732,261421777~3682745,272991490~2616824,280096820~3519235,290699362~6307058,299914309~5068798,313132291~1239533,336626861~6999065,355247735~1080322,367225159~6878947,386700378~10576664] wrote [11489330~2900752,28855924~6418790,159248657~3912237,215842100~4042443,244358165~5723321,251380213~412732,292749114~4257306,299914309~5068798,355247735~1080322,386700378~7240666] 2024-04-26T05:02:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:37 smithi005 ceph-mon[91801]: osdmap e919: 8 total, 8 up, 8 in 2024-04-26T05:02:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:37 smithi005 ceph-mon[95809]: osdmap e919: 8 total, 8 up, 8 in 2024-04-26T05:02:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:37 smithi111 ceph-mon[86360]: osdmap e919: 8 total, 8 up, 8 in 2024-04-26T05:02:38.905 INFO:tasks.workunit.client.0.smithi005.stdout: i=3 exists [2971003~7698347,11489330~2900752,28855924~6418790,38349730~1128998,51182602~7912451,66480843~8057474,81869044~2405596,103478426~1352441,154566487~8594407,180492905~1175234,197668267~4159401,215842100~4042443,226302548~7648367,244358165~5723321,251380213~5396239,261188250~4880704,272911712~2696602,280096820~3519235,290699362~7452367,299914309~5068798,313132291~1239533,336626861~6999065,354977002~7284179,367225159~6878947,386700378~10576664,403822911~8182071] wrote [66480843~5801200,154566487~7888195,197668267~4159401,251559371~5217081,261188250~4880704,272911712~1975472,293132346~5019383,354977002~7284179,392439153~1666787,403822911~8182071] 2024-04-26T05:02:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:38 smithi005 ceph-mon[91801]: pgmap v1069: 105 pgs: 105 active+clean; 87 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 12 MiB/s wr, 189 op/s 2024-04-26T05:02:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:38 smithi005 ceph-mon[91801]: osdmap e920: 8 total, 8 up, 8 in 2024-04-26T05:02:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:38 smithi005 ceph-mon[95809]: pgmap v1069: 105 pgs: 105 active+clean; 87 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 12 MiB/s wr, 189 op/s 2024-04-26T05:02:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:38 smithi005 ceph-mon[95809]: osdmap e920: 8 total, 8 up, 8 in 2024-04-26T05:02:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:38 smithi111 ceph-mon[86360]: pgmap v1069: 105 pgs: 105 active+clean; 87 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 12 MiB/s wr, 189 op/s 2024-04-26T05:02:39.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:38 smithi111 ceph-mon[86360]: osdmap e920: 8 total, 8 up, 8 in 2024-04-26T05:02:39.932 INFO:tasks.workunit.client.0.smithi005.stdout: i=4 exists [2971003~7698347,11489330~2900752,19655922~4622550,28855924~6418790,37176689~9510986,51182602~7912451,66480843~8057474,81869044~2405596,103478426~1352441,154566487~8594407,180492905~1175234,197668267~4159401,215842100~4042443,226302548~7648367,244358165~5723321,251380213~5396239,261188250~4880704,272911712~2696602,279687572~18464157,299914309~5068798,313132291~1239533,336626861~6999065,354977002~7284179,365305349~8798757,386700378~10576664,403822911~8182071] wrote [5233375~5360030,19655922~4622550,37176689~9510986,67139940~2012017,279687572~14589156,365305349~2273297] 2024-04-26T05:02:40.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:39 smithi005 ceph-mon[91801]: osdmap e921: 8 total, 8 up, 8 in 2024-04-26T05:02:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:39 smithi005 ceph-mon[95809]: osdmap e921: 8 total, 8 up, 8 in 2024-04-26T05:02:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:39 smithi111 ceph-mon[86360]: osdmap e921: 8 total, 8 up, 8 in 2024-04-26T05:02:40.910 INFO:tasks.workunit.client.0.smithi005.stdout: i=5 exists [302299~10367051,11489330~2900752,19655922~4622550,28855924~6418790,37116374~9571301,51182602~7912451,66480843~8057474,81869044~2405596,103478426~1352441,116565412~7685057,137521788~2700932,154566487~8594407,180492905~1175234,185111125~4467806,191193080~254549,191487735~1914156,197668267~4159401,215842100~4042443,226302548~7648367,244358165~5723321,251380213~5396239,261188250~4880704,272911712~2696602,279687572~18464157,299914309~5068798,313132291~1239533,336626861~6999065,354977002~7284179,365305349~8798757,386700378~10576664,403822911~8182071] wrote [302299~5546309,37116374~7993174,116565412~7685057,137521788~2700932,185111125~4467806,191193080~254549,191487735~1914156,284495798~1581943,290171585~1962174,387847638~8049474] 2024-04-26T05:02:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:40 smithi005 ceph-mon[91801]: pgmap v1072: 105 pgs: 105 active+clean; 87 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 12 MiB/s wr, 189 op/s 2024-04-26T05:02:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:40 smithi005 ceph-mon[91801]: osdmap e922: 8 total, 8 up, 8 in 2024-04-26T05:02:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:40 smithi005 ceph-mon[95809]: pgmap v1072: 105 pgs: 105 active+clean; 87 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 12 MiB/s wr, 189 op/s 2024-04-26T05:02:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:40 smithi005 ceph-mon[95809]: osdmap e922: 8 total, 8 up, 8 in 2024-04-26T05:02:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:40 smithi111 ceph-mon[86360]: pgmap v1072: 105 pgs: 105 active+clean; 87 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 12 MiB/s wr, 189 op/s 2024-04-26T05:02:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:40 smithi111 ceph-mon[86360]: osdmap e922: 8 total, 8 up, 8 in 2024-04-26T05:02:41.901 INFO:tasks.workunit.client.0.smithi005.stdout: i=6 exists [302299~10367051,11489330~2900752,19655922~4622550,28855924~6418790,37116374~9571301,51182602~7912451,66480843~8057474,81869044~2405596,93473157~6611621,103478426~1352441,116565412~7685057,136518793~7210842,153260641~9900253,166744138~5530070,180492905~1175234,184320730~5903453,191193080~254549,191487735~1914156,197668267~4159401,215842100~4042443,226302548~7648367,244358165~5723321,251380213~5396239,261188250~4880704,272911712~2696602,275875629~1326877,279687572~18464157,299914309~5068798,313132291~1239533,336626861~6999065,354977002~7284179,365305349~8798757,386700378~10576664,402068725~9936257] wrote [7984286~1691168,93473157~6611621,136518793~7210842,153260641~7209451,166744138~5530070,184320730~5903453,275875629~1326877,394462935~1230692,402068725~3177991] 2024-04-26T05:02:42.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:41 smithi005 ceph-mon[91801]: osdmap e923: 8 total, 8 up, 8 in 2024-04-26T05:02:42.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:41 smithi005 ceph-mon[95809]: osdmap e923: 8 total, 8 up, 8 in 2024-04-26T05:02:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:41 smithi111 ceph-mon[86360]: osdmap e923: 8 total, 8 up, 8 in 2024-04-26T05:02:42.918 INFO:tasks.workunit.client.0.smithi005.stdout: i=7 exists [302299~10367051,11489330~2900752,19655922~6470487,28855924~6418790,37116374~9571301,51182602~8126293,66480843~8057474,77085922~2980265,81869044~2405596,84832971~5474363,93473157~6611621,103478426~1352441,116565412~7685057,136518793~7210842,153260641~9900253,166744138~5530070,180492905~1175234,184320730~5903453,191193080~254549,191487735~1914156,197668267~4159401,215842100~4042443,226302548~7648367,234651902~8135742,244358165~12418287,261188250~4880704,272911712~2696602,275875629~1326877,279687572~18464157,299914309~5068798,313132291~1239533,314724252~2251118,318698963~7158522,336626861~6999065,354977002~7284179,365305349~8798757,386700378~10576664,402068725~9936257] wrote [22254791~3871618,52262560~7046335,77085922~2980265,84832971~5474363,167060994~2553031,232511991~631822,234651902~8135742,246386226~8168290,314724252~2251118,318698963~7158522] 2024-04-26T05:02:43.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:42 smithi005 ceph-mon[91801]: pgmap v1075: 105 pgs: 105 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 19 MiB/s wr, 201 op/s 2024-04-26T05:02:43.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:42 smithi005 ceph-mon[91801]: osdmap e924: 8 total, 8 up, 8 in 2024-04-26T05:02:43.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:42 smithi005 ceph-mon[95809]: pgmap v1075: 105 pgs: 105 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 19 MiB/s wr, 201 op/s 2024-04-26T05:02:43.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:42 smithi005 ceph-mon[95809]: osdmap e924: 8 total, 8 up, 8 in 2024-04-26T05:02:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:42 smithi111 ceph-mon[86360]: pgmap v1075: 105 pgs: 105 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 19 MiB/s wr, 201 op/s 2024-04-26T05:02:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:42 smithi111 ceph-mon[86360]: osdmap e924: 8 total, 8 up, 8 in 2024-04-26T05:02:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:02:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:02:43] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.43.0" 2024-04-26T05:02:43.922 INFO:tasks.workunit.client.0.smithi005.stdout: i=8 exists [302299~10854737,11489330~2900752,19655922~6470487,28855924~6418790,37116374~9571301,51182602~8126293,66480843~8057474,77085922~2980265,81869044~2405596,84832971~5474363,92160139~7924639,103478426~1352441,116565412~7685057,136518793~7210842,147175318~4385964,153260641~9900253,166643365~7933718,180492905~1175234,184320730~5903453,191193080~254549,191487735~1914156,197668267~4159401,215842100~18108815,234651902~8135742,244358165~12418287,261188250~4880704,272911712~2696602,275875629~1326877,279687572~18464157,299914309~5068798,313132291~1239533,314724252~2251118,318698963~7158522,336626861~6999065,354977002~7284179,365305349~8798757,376922969~7154736,386700378~10576664,400438529~910903,401373767~10631215] wrote [5269485~5887551,23283720~2132500,92160139~6849871,147175318~4385964,166643365~7933718,218572680~8669141,376922969~7154736,400438529~910903,401373767~762905] 2024-04-26T05:02:44.146 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:43 smithi005 ceph-mon[91801]: osdmap e925: 8 total, 8 up, 8 in 2024-04-26T05:02:44.146 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:43 smithi005 ceph-mon[95809]: osdmap e925: 8 total, 8 up, 8 in 2024-04-26T05:02:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:43 smithi111 ceph-mon[86360]: osdmap e925: 8 total, 8 up, 8 in 2024-04-26T05:02:44.934 INFO:tasks.workunit.client.0.smithi005.stdout: i=9 exists [302299~10854737,11489330~4269413,19655922~6470487,28855924~6418790,37116374~13719409,51182602~8126293,66480843~8057474,77085922~2980265,81869044~2405596,84832971~5474363,92160139~7924639,103478426~1352441,116565412~7685057,136518793~7210842,147175318~4385964,153260641~9900253,166643365~7933718,180492905~1175234,184320730~5903453,191193080~254549,191487735~1914156,197668267~4159401,215842100~18108815,234651902~8135742,244358165~12905030,261188250~4880704,272911712~2696602,275875629~1326877,279687572~18464157,299914309~5068798,313132291~1239533,314724252~2251118,318698963~7158522,336626861~6999065,344012199~7169161,354977002~7284179,365305349~8798757,376922969~7154736,386700378~10576664,400304034~11700948] wrote [783125~5460542,13561934~2196809,43254743~7581040,67676544~3990914,168890798~1748612,188464629~1274194,218772432~6044450,256211324~1051871,344012199~7169161,400304034~5666473] 2024-04-26T05:02:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:44 smithi005 ceph-mon[91801]: pgmap v1078: 105 pgs: 105 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 54 MiB/s wr, 286 op/s 2024-04-26T05:02:45.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:44 smithi005 ceph-mon[91801]: osdmap e926: 8 total, 8 up, 8 in 2024-04-26T05:02:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:44 smithi005 ceph-mon[95809]: pgmap v1078: 105 pgs: 105 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 54 MiB/s wr, 286 op/s 2024-04-26T05:02:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:44 smithi005 ceph-mon[95809]: osdmap e926: 8 total, 8 up, 8 in 2024-04-26T05:02:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:44 smithi111 ceph-mon[86360]: pgmap v1078: 105 pgs: 105 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 54 MiB/s wr, 286 op/s 2024-04-26T05:02:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:44 smithi111 ceph-mon[86360]: osdmap e926: 8 total, 8 up, 8 in 2024-04-26T05:02:45.933 INFO:tasks.workunit.client.0.smithi005.stdout: i=10 exists [302299~10854737,11489330~4269413,19655922~6470487,28855924~6418790,37116374~13719409,51182602~8126293,66480843~8057474,77085922~2980265,81869044~2405596,84832971~5474363,92160139~7924639,103478426~1352441,116565412~7685057,136518793~7210842,147175318~15985576,166643365~7933718,180492905~1175234,184320730~9081161,197668267~4159401,202445148~7342534,215842100~18108815,234651902~8135742,244358165~12905030,261188250~8120215,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~2251118,318698963~7158522,331093316~20088044,354977002~7284179,365305349~8798757,376922969~7154736,386700378~10576664,400304034~11700948] wrote [149691690~5506454,185766144~7577245,202445148~7342534,263695833~5612632,279240107~7900076,331093316~6441105,342334144~3368312,355392590~3316676] 2024-04-26T05:02:46.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:45 smithi005 ceph-mon[91801]: osdmap e927: 8 total, 8 up, 8 in 2024-04-26T05:02:46.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:45 smithi005 ceph-mon[95809]: osdmap e927: 8 total, 8 up, 8 in 2024-04-26T05:02:46.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:45 smithi111 ceph-mon[86360]: osdmap e927: 8 total, 8 up, 8 in 2024-04-26T05:02:46.930 INFO:tasks.workunit.client.0.smithi005.stdout: i=11 exists [302299~15456444,19655922~6470487,28855924~6418790,37116374~13719409,51182602~8126293,66480843~8057474,77085922~7715435,84832971~5474363,90339043~1406814,92160139~7924639,103478426~1352441,107775036~4051543,116565412~7685057,132625006~11104629,147175318~15985576,166643365~7933718,180492905~1175234,184320730~9081161,197668267~4159401,202445148~7342534,215842100~18108815,234651902~22611293,258371195~1295314,261188250~8120215,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~2251118,318086890~7770595,331093316~20088044,354977002~7284179,365305349~8798757,376922969~7154736,386700378~10576664,400304034~11700948] wrote [9439911~5590061,78783633~6017724,90339043~1406814,107775036~4051543,132625006~6553828,242663585~6221928,253775740~3258503,258371195~1295314,318086890~2062162,389673087~6285886] 2024-04-26T05:02:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:46 smithi005 ceph-mon[95809]: pgmap v1081: 105 pgs: 105 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 35 MiB/s wr, 85 op/s 2024-04-26T05:02:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:46 smithi005 ceph-mon[95809]: osdmap e928: 8 total, 8 up, 8 in 2024-04-26T05:02:47.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:46 smithi005 ceph-mon[91801]: pgmap v1081: 105 pgs: 105 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 35 MiB/s wr, 85 op/s 2024-04-26T05:02:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:46 smithi005 ceph-mon[91801]: osdmap e928: 8 total, 8 up, 8 in 2024-04-26T05:02:47.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:47.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:46 smithi111 ceph-mon[86360]: pgmap v1081: 105 pgs: 105 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 35 MiB/s wr, 85 op/s 2024-04-26T05:02:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:46 smithi111 ceph-mon[86360]: osdmap e928: 8 total, 8 up, 8 in 2024-04-26T05:02:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:47.948 INFO:tasks.workunit.client.0.smithi005.stdout: i=12 exists [302299~15456444,19655922~6470487,28855924~6418790,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,103478426~1352441,107775036~16475433,132625006~11104629,147175318~15985576,166643365~7933718,180492905~1175234,184320730~9081161,197668267~4159401,202445148~7342534,215842100~18108815,234651902~22611293,258371195~10937270,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~11133233,331093316~20088044,354977002~7284179,365305349~8798757,376922969~7154736,386700378~10576664,400304034~11700948] wrote [50475644~4266652,63814198~59916,67930327~12517,84216573~8426626,110871925~7907821,156449321~2530847,241148850~4540238,258719402~5239587,314931571~3495771] 2024-04-26T05:02:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:47 smithi005 ceph-mon[91801]: osdmap e929: 8 total, 8 up, 8 in 2024-04-26T05:02:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:47 smithi005 ceph-mon[95809]: osdmap e929: 8 total, 8 up, 8 in 2024-04-26T05:02:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:47 smithi111 ceph-mon[86360]: osdmap e929: 8 total, 8 up, 8 in 2024-04-26T05:02:48.960 INFO:tasks.workunit.client.0.smithi005.stdout: i=13 exists [302299~15456444,19655922~16050467,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,103478426~1352441,107775036~16475433,132625006~11104629,147175318~15985576,166643365~7933718,180492905~1175234,184320730~9081161,197668267~4159401,202445148~7342534,215842100~41421095,258371195~10937270,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~11133233,331093316~20088044,354977002~7284179,365305349~8798757,376334896~7742809,386700378~10576664,400304034~12499171] wrote [25394340~10312049,56376026~2372833,96315737~2355983,221294107~4925397,227555150~7962616,249467180~6387317,348926073~1766809,376334896~2280477,410713344~2089861] 2024-04-26T05:02:49.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:48 smithi005 ceph-mon[91801]: pgmap v1084: 105 pgs: 105 active+clean; 588 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 57 MiB/s wr, 343 op/s 2024-04-26T05:02:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:48 smithi005 ceph-mon[91801]: osdmap e930: 8 total, 8 up, 8 in 2024-04-26T05:02:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:48 smithi005 ceph-mon[95809]: pgmap v1084: 105 pgs: 105 active+clean; 588 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 57 MiB/s wr, 343 op/s 2024-04-26T05:02:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:48 smithi005 ceph-mon[95809]: osdmap e930: 8 total, 8 up, 8 in 2024-04-26T05:02:49.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:48 smithi111 ceph-mon[86360]: pgmap v1084: 105 pgs: 105 active+clean; 588 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 57 MiB/s wr, 343 op/s 2024-04-26T05:02:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:48 smithi111 ceph-mon[86360]: osdmap e930: 8 total, 8 up, 8 in 2024-04-26T05:02:49.974 INFO:tasks.workunit.client.0.smithi005.stdout: i=14 exists [302299~18936884,19655922~16050467,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,103478426~1352441,107775036~17299896,132625006~11104629,147175318~15985576,166643365~7933718,180492905~1175234,184320730~9081161,197668267~4159401,202445148~7342534,215842100~41421095,258371195~10937270,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~12352584,331093316~20088044,354977002~7284179,365305349~8798757,376334896~7742809,386700378~10576664,400304034~14484345] wrote [15405054~3834129,118865746~6209186,217407614~7756414,287827506~2816754,319573879~7502957,338848817~2692783,404811913~9976466] 2024-04-26T05:02:50.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:49 smithi005 ceph-mon[91801]: osdmap e931: 8 total, 8 up, 8 in 2024-04-26T05:02:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:49 smithi005 ceph-mon[95809]: osdmap e931: 8 total, 8 up, 8 in 2024-04-26T05:02:50.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:49 smithi111 ceph-mon[86360]: osdmap e931: 8 total, 8 up, 8 in 2024-04-26T05:02:50.973 INFO:tasks.workunit.client.0.smithi005.stdout: i=15 exists [302299~18936884,19655922~16050467,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,103478426~1352441,107775036~17299896,132625006~11104629,147175318~15985576,166643365~7933718,180492905~1175234,184320730~9081161,197668267~4159401,202445148~7342534,215842100~41421095,258371195~10937270,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~12352584,331093316~20088044,354977002~7284179,365305349~8798757,376334896~7742809,386700378~10576664,400304034~14484345] wrote [5767279~4537142,25122868~4031766,48924390~7170137,87280473~6160603,141226980~2451397,155115064~3314455,314906723~6906098,343763651~5010612,378780147~2202619] 2024-04-26T05:02:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:50 smithi005 ceph-mon[91801]: pgmap v1087: 105 pgs: 105 active+clean; 588 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 57 MiB/s wr, 343 op/s 2024-04-26T05:02:51.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:50 smithi005 ceph-mon[91801]: osdmap e932: 8 total, 8 up, 8 in 2024-04-26T05:02:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:50 smithi005 ceph-mon[95809]: pgmap v1087: 105 pgs: 105 active+clean; 588 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 57 MiB/s wr, 343 op/s 2024-04-26T05:02:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:50 smithi005 ceph-mon[95809]: osdmap e932: 8 total, 8 up, 8 in 2024-04-26T05:02:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:50 smithi111 ceph-mon[86360]: pgmap v1087: 105 pgs: 105 active+clean; 588 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 57 MiB/s wr, 343 op/s 2024-04-26T05:02:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:50 smithi111 ceph-mon[86360]: osdmap e932: 8 total, 8 up, 8 in 2024-04-26T05:02:51.979 INFO:tasks.workunit.client.0.smithi005.stdout: i=16 exists [302299~35404090,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,103478426~1352441,107775036~21253197,132625006~11104629,147175318~15985576,166643365~7933718,180492905~1175234,181856616~1980279,184320730~9081161,197668267~4159401,202445148~7342534,215842100~41421095,258371195~10937270,272911712~2696602,275875629~1326877,279240107~18911622,299914309~5068798,313132291~1239533,314724252~12352584,331093316~20088044,354977002~7284179,365305349~8798757,376334896~7742809,386700378~10576664,400304034~14484345] wrote [18452951~1791391,44310797~3851392,82049313~10041168,113922370~7444203,124735011~4293222,147383716~4688850,181856616~1980279,261634354~711054,320074641~6590827] 2024-04-26T05:02:52.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:51 smithi005 ceph-mon[91801]: osdmap e933: 8 total, 8 up, 8 in 2024-04-26T05:02:52.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:51 smithi005 ceph-mon[95809]: osdmap e933: 8 total, 8 up, 8 in 2024-04-26T05:02:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:51 smithi111 ceph-mon[86360]: osdmap e933: 8 total, 8 up, 8 in 2024-04-26T05:02:52.944 INFO:tasks.workunit.client.0.smithi005.stdout: i=17 exists [302299~35404090,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,101890038~861819,103478426~1352441,107775036~21253197,132625006~11104629,147175318~17166498,166643365~7933718,180492905~1175234,181856616~1980279,184320730~9081161,197668267~4159401,202445148~7342534,215842100~41549371,258371195~10937270,272911712~2696602,275875629~1326877,278814181~19337548,299914309~5068798,313132291~1239533,314724252~12352584,331093316~20088044,354977002~7284179,365305349~8798757,376334896~9482752,386700378~10576664,400304034~14484345] wrote [78916945~5452996,101890038~861819,123956676~1844468,157557257~6784559,185672831~596121,253993910~3397561,278814181~2428109,290123339~1158954,323323361~2786808,383248028~2569620] 2024-04-26T05:02:53.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:52 smithi005 ceph-mon[91801]: pgmap v1090: 105 pgs: 105 active+clean; 654 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 120 KiB/s rd, 16 MiB/s wr, 201 op/s 2024-04-26T05:02:53.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:52 smithi005 ceph-mon[91801]: osdmap e934: 8 total, 8 up, 8 in 2024-04-26T05:02:53.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:52 smithi005 ceph-mon[95809]: pgmap v1090: 105 pgs: 105 active+clean; 654 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 120 KiB/s rd, 16 MiB/s wr, 201 op/s 2024-04-26T05:02:53.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:52 smithi005 ceph-mon[95809]: osdmap e934: 8 total, 8 up, 8 in 2024-04-26T05:02:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:52 smithi111 ceph-mon[86360]: pgmap v1090: 105 pgs: 105 active+clean; 654 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 120 KiB/s rd, 16 MiB/s wr, 201 op/s 2024-04-26T05:02:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:52 smithi111 ceph-mon[86360]: osdmap e934: 8 total, 8 up, 8 in 2024-04-26T05:02:53.481 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:02:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:02:53] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.43.0" 2024-04-26T05:02:53.982 INFO:tasks.workunit.client.0.smithi005.stdout: i=18 exists [302299~35404090,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,101890038~861819,103478426~1352441,107775036~21253197,132625006~11104629,147175318~17166498,166643365~7933718,180492905~1175234,181856616~1980279,184320730~9081161,193574292~8253376,202186289~7601393,215842100~53466365,272911712~2696602,275875629~1326877,278814181~26168926,313132291~1239533,314724252~12352584,331093316~20088044,354977002~7284179,365305349~8798757,376334896~9482752,386700378~10576664,400304034~14484345] wrote [25317367~8165380,44636554~2021639,68157190~1012229,123036883~3023917,147712759~2424628,193574292~6698852,202186289~2000112,254932986~7502872,298047680~4479086,324479544~776728] 2024-04-26T05:02:54.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:53 smithi005 ceph-mon[91801]: osdmap e935: 8 total, 8 up, 8 in 2024-04-26T05:02:54.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:53 smithi005 ceph-mon[95809]: osdmap e935: 8 total, 8 up, 8 in 2024-04-26T05:02:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:53 smithi111 ceph-mon[86360]: osdmap e935: 8 total, 8 up, 8 in 2024-04-26T05:02:54.974 INFO:tasks.workunit.client.0.smithi005.stdout: i=19 exists [302299~35404090,37116374~22192521,63814198~59916,66480843~8057474,77085922~22998856,101890038~2940829,107775036~22140126,132625006~11104629,147175318~17166498,166643365~7933718,180492905~1175234,181856616~1980279,184320730~9081161,193574292~8253376,202186289~7601393,215842100~53466365,272911712~2696602,275875629~1326877,278814181~26168926,313132291~1239533,314724252~12352584,331093316~23745628,354977002~7284179,365305349~8798757,376334896~9482752,386700378~10576664,400304034~14484345] wrote [11976194~2429408,51260220~4928884,69276111~548914,102525700~1239898,123805941~6109221,169906616~3204684,231651653~6542992,259324931~4455630,348669601~6169343,402743139~4350173] 2024-04-26T05:02:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:54 smithi005 ceph-mon[91801]: pgmap v1093: 105 pgs: 105 active+clean; 776 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 236 KiB/s rd, 49 MiB/s wr, 372 op/s 2024-04-26T05:02:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:54 smithi005 ceph-mon[91801]: osdmap e936: 8 total, 8 up, 8 in 2024-04-26T05:02:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:54 smithi005 ceph-mon[95809]: pgmap v1093: 105 pgs: 105 active+clean; 776 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 236 KiB/s rd, 49 MiB/s wr, 372 op/s 2024-04-26T05:02:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:54 smithi005 ceph-mon[95809]: osdmap e936: 8 total, 8 up, 8 in 2024-04-26T05:02:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:54 smithi111 ceph-mon[86360]: pgmap v1093: 105 pgs: 105 active+clean; 776 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 236 KiB/s rd, 49 MiB/s wr, 372 op/s 2024-04-26T05:02:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:54 smithi111 ceph-mon[86360]: osdmap e936: 8 total, 8 up, 8 in 2024-04-26T05:02:55.936 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 1 diff [0~4194304,37748736~4194304,50331648~8388608,100663296~4194304,180355072~4194304,222298112~8388608,272629760~4194304,339738624~4194304,364904448~8388608] 2024-04-26T05:02:55.936 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~4194304,37748736~4194304,50331648~8388608,100663296~4194304,180355072~4194304,222298112~8388608,272629760~4194304,339738624~4194304,364904448~8388608] 2024-04-26T05:02:55.948 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~4194304,37748736~4194304,50331648~12582912,100663296~4194304,180355072~4194304,222298112~12582912,272629760~4194304,339738624~4194304,364904448~12582912] 2024-04-26T05:02:55.948 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 2 diff [0~4194304,8388608~4194304,25165824~8388608,37748736~4194304,50331648~8388608,100663296~4194304,155189248~4194304,180355072~4194304,213909504~4194304,222298112~8388608,243269632~8388608,272629760~4194304,289406976~16777216,339738624~4194304,352321536~4194304,364904448~8388608,385875968~8388608] 2024-04-26T05:02:55.948 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~4194304,8388608~4194304,25165824~8388608,37748736~4194304,50331648~8388608,100663296~4194304,155189248~4194304,180355072~4194304,213909504~4194304,222298112~8388608,243269632~8388608,272629760~4194304,289406976~16777216,339738624~4194304,352321536~4194304,364904448~8388608,385875968~8388608] 2024-04-26T05:02:55.950 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~4194304,8388608~8388608,25165824~16777216,50331648~12582912,100663296~4194304,155189248~8388608,180355072~4194304,213909504~20971520,243269632~12582912,272629760~4194304,289406976~16777216,339738624~4194304,352321536~4194304,364904448~12582912,385875968~8388608] 2024-04-26T05:02:55.950 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 3 diff [0~4194304,8388608~4194304,25165824~8388608,37748736~4194304,50331648~8388608,62914560~8388608,100663296~4194304,150994944~12582912,180355072~4194304,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~4194304,289406976~16777216,339738624~4194304,352321536~8388608,364904448~8388608,385875968~8388608,402653184~8388608] 2024-04-26T05:02:55.950 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~4194304,8388608~4194304,25165824~8388608,37748736~4194304,50331648~8388608,62914560~8388608,100663296~4194304,150994944~12582912,180355072~4194304,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~4194304,289406976~16777216,339738624~4194304,352321536~8388608,364904448~8388608,385875968~8388608,402653184~8388608] 2024-04-26T05:02:55.953 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~4194304,8388608~8388608,25165824~16777216,50331648~25165824,100663296~4194304,150994944~12582912,180355072~4194304,197132288~8388608,213909504~20971520,243269632~25165824,272629760~4194304,289406976~16777216,339738624~4194304,352321536~25165824,385875968~8388608,402653184~12582912] 2024-04-26T05:02:55.953 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 4 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,100663296~4194304,150994944~12582912,180355072~4194304,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~33554432,339738624~4194304,352321536~8388608,364904448~12582912,385875968~8388608,402653184~8388608] 2024-04-26T05:02:55.953 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,100663296~4194304,150994944~12582912,180355072~4194304,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~33554432,339738624~4194304,352321536~8388608,364904448~12582912,385875968~8388608,402653184~8388608] 2024-04-26T05:02:55.956 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~75497472,100663296~4194304,150994944~12582912,180355072~4194304,197132288~8388608,213909504~20971520,243269632~25165824,272629760~33554432,339738624~4194304,352321536~25165824,385875968~8388608,402653184~12582912] 2024-04-26T05:02:55.956 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 5 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,100663296~4194304,113246208~8388608,134217728~4194304,150994944~12582912,180355072~12582912,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~33554432,339738624~4194304,352321536~8388608,364904448~12582912,385875968~12582912,402653184~8388608] 2024-04-26T05:02:55.956 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,100663296~4194304,113246208~8388608,134217728~4194304,150994944~12582912,180355072~12582912,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~33554432,339738624~4194304,352321536~8388608,364904448~12582912,385875968~12582912,402653184~8388608] 2024-04-26T05:02:55.959 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~75497472,100663296~4194304,113246208~12582912,134217728~8388608,150994944~12582912,180355072~25165824,213909504~20971520,243269632~25165824,272629760~33554432,339738624~4194304,352321536~25165824,385875968~12582912,402653184~12582912] 2024-04-26T05:02:55.959 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 6 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,92274688~12582912,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~33554432,339738624~4194304,352321536~8388608,364904448~12582912,385875968~25165824] 2024-04-26T05:02:55.959 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,92274688~12582912,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,222298112~8388608,243269632~12582912,260046848~8388608,272629760~33554432,339738624~4194304,352321536~8388608,364904448~12582912,385875968~25165824] 2024-04-26T05:02:55.964 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~75497472,92274688~12582912,113246208~12582912,134217728~12582912,150994944~25165824,180355072~25165824,213909504~20971520,243269632~25165824,272629760~33554432,339738624~4194304,352321536~25165824,385875968~29360128] 2024-04-26T05:02:55.964 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 7 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~20971520,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,222298112~16777216,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,339738624~4194304,352321536~8388608,364904448~12582912,385875968~25165824] 2024-04-26T05:02:55.964 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~20971520,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,222298112~16777216,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,339738624~4194304,352321536~8388608,364904448~12582912,385875968~25165824] 2024-04-26T05:02:55.968 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~104857600,113246208~12582912,134217728~12582912,150994944~25165824,180355072~25165824,213909504~54525952,272629760~33554432,314572800~12582912,339738624~4194304,352321536~25165824,385875968~29360128] 2024-04-26T05:02:55.969 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 8 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~12582912,100663296~4194304,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~25165824,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,339738624~4194304,352321536~8388608,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.969 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~12582912,100663296~4194304,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~25165824,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,339738624~4194304,352321536~8388608,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.974 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~104857600,113246208~12582912,134217728~41943040,180355072~25165824,213909504~54525952,272629760~33554432,314572800~12582912,339738624~4194304,352321536~62914560] 2024-04-26T05:02:55.974 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 9 diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,100663296~4194304,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~25165824,243269632~25165824,272629760~33554432,314572800~8388608,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.974 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,100663296~4194304,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~25165824,243269632~25165824,272629760~33554432,314572800~8388608,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.981 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~104857600,113246208~12582912,134217728~41943040,180355072~25165824,213909504~54525952,272629760~33554432,314572800~12582912,339738624~75497472] 2024-04-26T05:02:55.981 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 10 diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,100663296~4194304,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~12582912,213909504~25165824,243269632~25165824,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.981 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,100663296~4194304,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~12582912,213909504~25165824,243269632~25165824,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.987 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~104857600,113246208~12582912,134217728~41943040,180355072~125829120,314572800~100663296] 2024-04-26T05:02:55.988 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 11 diff [0~58720256,62914560~8388608,75497472~20971520,100663296~8388608,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.988 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~20971520,100663296~8388608,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.994 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~125829120,130023424~46137344,180355072~125829120,314572800~100663296] 2024-04-26T05:02:55.994 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 12 diff [0~58720256,62914560~8388608,75497472~46137344,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~20971520,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:55.994 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~46137344,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~20971520,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:56.001 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~125829120,130023424~46137344,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.001 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 13 diff [0~58720256,62914560~8388608,75497472~46137344,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~20971520,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:56.001 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~46137344,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~20971520,339738624~20971520,364904448~16777216,385875968~25165824] 2024-04-26T05:02:56.007 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~125829120,130023424~46137344,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.007 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 14 diff [0~58720256,62914560~8388608,75497472~50331648,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~16777216,385875968~29360128] 2024-04-26T05:02:56.007 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~50331648,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~16777216,385875968~29360128] 2024-04-26T05:02:56.016 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~125829120,130023424~46137344,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.016 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 15 diff [0~58720256,62914560~8388608,75497472~50331648,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~16777216,385875968~29360128] 2024-04-26T05:02:56.016 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~50331648,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~16777216,385875968~29360128] 2024-04-26T05:02:56.023 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~125829120,130023424~46137344,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.023 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 16 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~16777216,385875968~29360128] 2024-04-26T05:02:56.023 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~16777216,385875968~29360128] 2024-04-26T05:02:56.032 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.032 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 17 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~50331648] 2024-04-26T05:02:56.032 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~50331648] 2024-04-26T05:02:56.041 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.041 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 18 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~50331648] 2024-04-26T05:02:56.041 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~33554432,352321536~8388608,364904448~50331648] 2024-04-26T05:02:56.051 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.051 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to 19 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:56.051 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:56.060 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.060 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 2 diff [8388608~4194304,25165824~8388608,155189248~4194304,213909504~4194304,243269632~8388608,289406976~16777216,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.060 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,155189248~4194304,213909504~4194304,243269632~8388608,289406976~16777216,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.062 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~12582912,155189248~8388608,213909504~8388608,243269632~12582912,289406976~16777216,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.062 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 3 diff [8388608~4194304,25165824~8388608,62914560~8388608,150994944~12582912,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~4194304,289406976~16777216,352321536~8388608,385875968~8388608,402653184~8388608] 2024-04-26T05:02:56.062 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,62914560~8388608,150994944~12582912,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~4194304,289406976~16777216,352321536~8388608,385875968~8388608,402653184~8388608] 2024-04-26T05:02:56.063 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~12582912,62914560~12582912,150994944~12582912,197132288~8388608,213909504~8388608,243269632~25165824,272629760~4194304,289406976~16777216,352321536~12582912,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.063 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 4 diff [4194304~8388608,16777216~29360128,62914560~8388608,150994944~12582912,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~33554432,352321536~8388608,364904448~4194304,385875968~8388608,402653184~8388608] 2024-04-26T05:02:56.063 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~29360128,62914560~8388608,150994944~12582912,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~33554432,352321536~8388608,364904448~4194304,385875968~8388608,402653184~8388608] 2024-04-26T05:02:56.065 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~46137344,62914560~12582912,150994944~12582912,197132288~8388608,213909504~8388608,243269632~25165824,272629760~33554432,352321536~16777216,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.065 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 5 diff [0~12582912,16777216~29360128,62914560~8388608,113246208~8388608,134217728~4194304,150994944~12582912,184549376~8388608,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~33554432,352321536~8388608,364904448~4194304,385875968~12582912,402653184~8388608] 2024-04-26T05:02:56.065 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,62914560~8388608,113246208~8388608,134217728~4194304,150994944~12582912,184549376~8388608,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~33554432,352321536~8388608,364904448~4194304,385875968~12582912,402653184~8388608] 2024-04-26T05:02:56.068 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~50331648,62914560~12582912,113246208~12582912,134217728~8388608,150994944~12582912,184549376~20971520,213909504~8388608,243269632~25165824,272629760~33554432,352321536~16777216,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.068 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 6 diff [0~12582912,16777216~29360128,62914560~8388608,92274688~8388608,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~33554432,352321536~8388608,364904448~4194304,385875968~25165824] 2024-04-26T05:02:56.069 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,62914560~8388608,92274688~8388608,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,243269632~12582912,260046848~8388608,272629760~33554432,352321536~8388608,364904448~4194304,385875968~25165824] 2024-04-26T05:02:56.071 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~50331648,62914560~12582912,92274688~8388608,113246208~12582912,134217728~12582912,150994944~25165824,180355072~25165824,213909504~8388608,243269632~25165824,272629760~33554432,352321536~16777216,385875968~29360128] 2024-04-26T05:02:56.072 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 7 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,230686720~8388608,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,352321536~8388608,364904448~4194304,385875968~25165824] 2024-04-26T05:02:56.072 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,213909504~4194304,230686720~8388608,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,352321536~8388608,364904448~4194304,385875968~25165824] 2024-04-26T05:02:56.076 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,113246208~12582912,134217728~12582912,150994944~25165824,180355072~25165824,213909504~8388608,230686720~37748736,272629760~33554432,314572800~12582912,352321536~16777216,385875968~29360128] 2024-04-26T05:02:56.076 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 8 diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~12582912,230686720~8388608,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,352321536~8388608,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.076 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~29360128,50331648~8388608,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~12582912,230686720~8388608,243269632~12582912,260046848~8388608,272629760~33554432,314572800~8388608,352321536~8388608,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.079 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,113246208~12582912,134217728~41943040,180355072~25165824,213909504~54525952,272629760~33554432,314572800~12582912,352321536~16777216,373293056~41943040] 2024-04-26T05:02:56.080 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 9 diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~12582912,230686720~8388608,243269632~25165824,272629760~33554432,314572800~8388608,343932928~16777216,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.080 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,213909504~12582912,230686720~8388608,243269632~25165824,272629760~33554432,314572800~8388608,343932928~16777216,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.084 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,113246208~12582912,134217728~41943040,180355072~25165824,213909504~54525952,272629760~33554432,314572800~12582912,343932928~25165824,373293056~41943040] 2024-04-26T05:02:56.084 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 10 diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~12582912,213909504~12582912,230686720~8388608,243269632~25165824,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.085 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~12582912,213909504~12582912,230686720~8388608,243269632~25165824,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.089 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,113246208~12582912,134217728~41943040,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.089 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 11 diff [0~58720256,62914560~8388608,75497472~20971520,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~12582912,230686720~37748736,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.090 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~20971520,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~12582912,230686720~37748736,272629760~33554432,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.095 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.095 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 12 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~12582912,230686720~37748736,272629760~33554432,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.095 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~12582912,230686720~37748736,272629760~33554432,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.100 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.100 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 13 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.100 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~25165824] 2024-04-26T05:02:56.106 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.106 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 14 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.106 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.112 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.112 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 15 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.112 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.119 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.119 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 16 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.119 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.125 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~71303168,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.125 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 17 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.125 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.133 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.133 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 18 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.133 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.142 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.142 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to 19 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.142 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.149 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.149 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 3 diff [62914560~8388608,150994944~8388608,197132288~4194304,247463936~8388608,260046848~8388608,272629760~4194304,289406976~8388608,352321536~8388608,390070272~4194304,402653184~8388608] 2024-04-26T05:02:56.149 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [62914560~8388608,150994944~8388608,197132288~4194304,247463936~8388608,260046848~8388608,272629760~4194304,289406976~8388608,352321536~8388608,390070272~4194304,402653184~8388608] 2024-04-26T05:02:56.150 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [62914560~12582912,150994944~12582912,197132288~8388608,247463936~20971520,272629760~4194304,289406976~12582912,352321536~12582912,390070272~4194304,402653184~12582912] 2024-04-26T05:02:56.150 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 4 diff [4194304~8388608,16777216~8388608,33554432~12582912,62914560~8388608,150994944~8388608,197132288~4194304,247463936~8388608,260046848~8388608,272629760~25165824,352321536~8388608,364904448~4194304,390070272~4194304,402653184~8388608] 2024-04-26T05:02:56.151 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~8388608,33554432~12582912,62914560~8388608,150994944~8388608,197132288~4194304,247463936~8388608,260046848~8388608,272629760~25165824,352321536~8388608,364904448~4194304,390070272~4194304,402653184~8388608] 2024-04-26T05:02:56.153 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,16777216~8388608,33554432~16777216,62914560~12582912,150994944~12582912,197132288~8388608,247463936~20971520,272629760~29360128,352321536~16777216,390070272~4194304,402653184~12582912] 2024-04-26T05:02:56.153 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 5 diff [0~12582912,16777216~8388608,33554432~12582912,62914560~8388608,113246208~8388608,134217728~4194304,150994944~8388608,184549376~8388608,197132288~4194304,247463936~8388608,260046848~8388608,272629760~25165824,352321536~8388608,364904448~4194304,385875968~8388608,402653184~8388608] 2024-04-26T05:02:56.153 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,62914560~8388608,113246208~8388608,134217728~4194304,150994944~8388608,184549376~8388608,197132288~4194304,247463936~8388608,260046848~8388608,272629760~25165824,352321536~8388608,364904448~4194304,385875968~8388608,402653184~8388608] 2024-04-26T05:02:56.155 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~8388608,33554432~16777216,62914560~12582912,113246208~12582912,134217728~8388608,150994944~12582912,184549376~20971520,247463936~20971520,272629760~29360128,352321536~16777216,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.155 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 6 diff [0~12582912,16777216~8388608,33554432~12582912,62914560~8388608,92274688~8388608,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,247463936~8388608,260046848~8388608,272629760~25165824,352321536~8388608,364904448~4194304,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.155 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,62914560~8388608,92274688~8388608,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,247463936~8388608,260046848~8388608,272629760~25165824,352321536~8388608,364904448~4194304,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.158 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~8388608,33554432~16777216,62914560~12582912,92274688~8388608,113246208~12582912,134217728~12582912,150994944~25165824,180355072~25165824,247463936~20971520,272629760~29360128,352321536~16777216,385875968~29360128] 2024-04-26T05:02:56.158 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 7 diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,62914560~8388608,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,230686720~8388608,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,352321536~8388608,364904448~4194304,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.158 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,62914560~8388608,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~20971520,180355072~12582912,197132288~4194304,230686720~8388608,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,352321536~8388608,364904448~4194304,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.160 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~12582912,33554432~67108864,113246208~12582912,134217728~12582912,150994944~25165824,180355072~25165824,230686720~37748736,272629760~29360128,314572800~12582912,352321536~16777216,385875968~29360128] 2024-04-26T05:02:56.160 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 8 diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,218103808~20971520,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,352321536~8388608,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.160 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,218103808~20971520,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,352321536~8388608,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.164 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~12582912,33554432~67108864,113246208~12582912,134217728~41943040,180355072~25165824,218103808~50331648,272629760~29360128,314572800~12582912,352321536~16777216,373293056~41943040] 2024-04-26T05:02:56.164 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 9 diff [0~25165824,33554432~25165824,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,218103808~20971520,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,343932928~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.164 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~4194304,218103808~20971520,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,343932928~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.167 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~67108864,113246208~12582912,134217728~41943040,180355072~25165824,218103808~50331648,272629760~29360128,314572800~12582912,343932928~25165824,373293056~41943040] 2024-04-26T05:02:56.167 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 10 diff [0~25165824,33554432~25165824,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~12582912,218103808~20971520,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.167 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,62914560~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,197132288~12582912,218103808~20971520,243269632~12582912,260046848~8388608,272629760~25165824,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.171 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~67108864,113246208~12582912,134217728~41943040,180355072~33554432,218103808~83886080,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.171 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 11 diff [0~25165824,33554432~25165824,62914560~8388608,75497472~20971520,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,218103808~50331648,272629760~25165824,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.171 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,62914560~8388608,75497472~20971520,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,218103808~50331648,272629760~25165824,314572800~8388608,327155712~8388608,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.175 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~67108864,104857600~20971520,130023424~46137344,180355072~33554432,218103808~83886080,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.175 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 12 diff [0~25165824,33554432~25165824,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,218103808~50331648,272629760~25165824,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.175 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,218103808~50331648,272629760~25165824,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.180 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~67108864,104857600~20971520,130023424~46137344,180355072~33554432,218103808~83886080,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.180 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 13 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,218103808~50331648,272629760~25165824,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.180 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,218103808~50331648,272629760~25165824,314572800~20971520,339738624~20971520,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.185 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~33554432,218103808~83886080,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.185 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 14 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.185 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.190 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~121634816,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.190 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 15 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.190 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.196 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~121634816,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.196 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 16 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.196 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.202 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~71303168,180355072~121634816,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.202 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 17 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.202 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,197132288~12582912,213909504~54525952,272629760~25165824,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.207 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~121634816,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.208 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 18 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.208 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.215 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.215 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to 19 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.215 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.222 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.222 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 4 diff [4194304~8388608,16777216~8388608,33554432~12582912,67108864~4194304,276824064~16777216,364904448~4194304] 2024-04-26T05:02:56.222 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~8388608,33554432~12582912,67108864~4194304,276824064~16777216,364904448~4194304] 2024-04-26T05:02:56.223 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,16777216~8388608,33554432~16777216,67108864~4194304,276824064~20971520,364904448~4194304] 2024-04-26T05:02:56.223 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 5 diff [0~12582912,16777216~8388608,33554432~12582912,67108864~4194304,113246208~8388608,134217728~4194304,184549376~8388608,276824064~16777216,364904448~4194304,385875968~8388608] 2024-04-26T05:02:56.224 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,67108864~4194304,113246208~8388608,134217728~4194304,184549376~8388608,276824064~16777216,364904448~4194304,385875968~8388608] 2024-04-26T05:02:56.225 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~8388608,33554432~16777216,67108864~4194304,113246208~12582912,134217728~8388608,184549376~12582912,276824064~20971520,364904448~4194304,385875968~12582912] 2024-04-26T05:02:56.225 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 6 diff [0~12582912,16777216~8388608,33554432~12582912,67108864~4194304,92274688~8388608,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,272629760~20971520,364904448~4194304,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.225 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,67108864~4194304,92274688~8388608,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,272629760~20971520,364904448~4194304,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.227 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~8388608,33554432~16777216,67108864~4194304,92274688~8388608,113246208~12582912,134217728~12582912,150994944~25165824,180355072~16777216,272629760~25165824,364904448~4194304,385875968~20971520] 2024-04-26T05:02:56.227 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 7 diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,67108864~4194304,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,230686720~8388608,243269632~8388608,272629760~20971520,314572800~8388608,364904448~4194304,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.227 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,67108864~4194304,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,230686720~8388608,243269632~8388608,272629760~20971520,314572800~8388608,364904448~4194304,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.229 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~12582912,33554432~29360128,67108864~4194304,75497472~25165824,113246208~12582912,134217728~12582912,150994944~25165824,180355072~16777216,230686720~25165824,272629760~25165824,314572800~12582912,364904448~4194304,385875968~20971520] 2024-04-26T05:02:56.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:55 smithi005 ceph-mon[91801]: osdmap e937: 8 total, 8 up, 8 in 2024-04-26T05:02:56.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:55 smithi005 ceph-mon[95809]: osdmap e937: 8 total, 8 up, 8 in 2024-04-26T05:02:56.230 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 8 diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,67108864~4194304,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,272629760~20971520,314572800~8388608,364904448~4194304,373293056~8388608,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.230 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,16777216~8388608,33554432~12582912,50331648~8388608,67108864~4194304,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,272629760~20971520,314572800~8388608,364904448~4194304,373293056~8388608,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.232 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,16777216~12582912,33554432~29360128,67108864~4194304,75497472~25165824,113246208~12582912,134217728~41943040,180355072~16777216,218103808~37748736,272629760~25165824,314572800~12582912,364904448~4194304,373293056~33554432] 2024-04-26T05:02:56.233 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 9 diff [0~25165824,33554432~25165824,67108864~12582912,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,255852544~4194304,272629760~20971520,314572800~8388608,343932928~8388608,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.233 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,67108864~12582912,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,255852544~4194304,272629760~20971520,314572800~8388608,343932928~8388608,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.235 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~29360128,67108864~33554432,113246208~12582912,134217728~41943040,180355072~16777216,218103808~41943040,272629760~25165824,314572800~12582912,343932928~8388608,364904448~4194304,373293056~33554432] 2024-04-26T05:02:56.235 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 10 diff [0~25165824,33554432~25165824,67108864~12582912,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,272629760~20971520,314572800~8388608,327155712~8388608,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.235 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,67108864~12582912,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,272629760~20971520,314572800~8388608,327155712~8388608,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.238 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~29360128,67108864~33554432,113246208~12582912,134217728~41943040,180355072~16777216,201326592~12582912,218103808~79691776,314572800~46137344,364904448~4194304,373293056~33554432] 2024-04-26T05:02:56.238 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 11 diff [0~25165824,33554432~25165824,67108864~29360128,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~20971520,314572800~8388608,327155712~8388608,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.238 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,67108864~29360128,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~20971520,314572800~8388608,327155712~8388608,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.242 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~29360128,67108864~33554432,104857600~20971520,130023424~46137344,180355072~16777216,201326592~12582912,218103808~79691776,314572800~46137344,364904448~4194304,373293056~33554432] 2024-04-26T05:02:56.242 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 12 diff [0~25165824,33554432~25165824,62914560~37748736,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~20971520,314572800~20971520,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.242 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~25165824,33554432~25165824,62914560~37748736,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~20971520,314572800~20971520,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.246 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~29360128,33554432~67108864,104857600~20971520,130023424~46137344,180355072~16777216,201326592~12582912,218103808~79691776,314572800~46137344,364904448~4194304,373293056~33554432] 2024-04-26T05:02:56.246 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 13 diff [0~58720256,62914560~37748736,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~20971520,314572800~20971520,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.246 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~37748736,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~20971520,314572800~20971520,339738624~16777216,364904448~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.250 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~16777216,201326592~12582912,218103808~79691776,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.250 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 14 diff [0~58720256,62914560~37748736,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.250 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~37748736,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.254 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~16777216,201326592~96468992,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.255 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 15 diff [0~58720256,62914560~37748736,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.255 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~37748736,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.260 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~16777216,201326592~96468992,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.260 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 16 diff [0~58720256,62914560~37748736,104857600~37748736,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.260 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~37748736,104857600~37748736,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.265 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~71303168,180355072~16777216,201326592~96468992,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.265 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 17 diff [0~58720256,62914560~79691776,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.266 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:55 smithi111 ceph-mon[86360]: osdmap e937: 8 total, 8 up, 8 in 2024-04-26T05:02:56.272 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~16777216,201326592~96468992,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.272 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 18 diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.272 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.278 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.278 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to 19 diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.278 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.285 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.285 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 5 diff [0~8388608,33554432~8388608,113246208~8388608,134217728~4194304,184549376~8388608,281018368~4194304,289406976~4194304,385875968~8388608] 2024-04-26T05:02:56.285 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~8388608,33554432~8388608,113246208~8388608,134217728~4194304,184549376~8388608,281018368~4194304,289406976~4194304,385875968~8388608] 2024-04-26T05:02:56.286 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~8388608,33554432~12582912,113246208~12582912,134217728~8388608,184549376~12582912,281018368~12582912,385875968~12582912] 2024-04-26T05:02:56.286 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 6 diff [0~8388608,33554432~8388608,92274688~8388608,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,272629760~4194304,281018368~4194304,289406976~4194304,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.286 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~8388608,33554432~8388608,92274688~8388608,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,272629760~4194304,281018368~4194304,289406976~4194304,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.290 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,33554432~12582912,92274688~8388608,113246208~12582912,134217728~12582912,150994944~25165824,180355072~16777216,272629760~20971520,385875968~20971520] 2024-04-26T05:02:56.290 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 7 diff [0~8388608,20971520~4194304,33554432~8388608,50331648~8388608,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,230686720~8388608,243269632~8388608,272629760~4194304,281018368~4194304,289406976~4194304,314572800~8388608,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.290 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~8388608,20971520~4194304,33554432~8388608,50331648~8388608,75497472~4194304,83886080~16777216,113246208~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~12582912,230686720~8388608,243269632~8388608,272629760~4194304,281018368~4194304,289406976~4194304,314572800~8388608,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.290 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,20971520~8388608,33554432~12582912,50331648~12582912,75497472~25165824,113246208~12582912,134217728~12582912,150994944~25165824,180355072~16777216,230686720~25165824,272629760~20971520,314572800~12582912,385875968~20971520] 2024-04-26T05:02:56.290 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 8 diff [0~12582912,20971520~4194304,33554432~8388608,50331648~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,272629760~4194304,281018368~4194304,289406976~4194304,314572800~8388608,373293056~8388608,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.290 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~12582912,20971520~4194304,33554432~8388608,50331648~8388608,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,272629760~4194304,281018368~4194304,289406976~4194304,314572800~8388608,373293056~8388608,385875968~8388608,398458880~4194304] 2024-04-26T05:02:56.294 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~12582912,20971520~8388608,33554432~12582912,50331648~12582912,75497472~25165824,113246208~12582912,134217728~41943040,180355072~16777216,218103808~37748736,272629760~20971520,314572800~12582912,373293056~33554432] 2024-04-26T05:02:56.294 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 9 diff [0~16777216,20971520~4194304,33554432~25165824,67108864~4194304,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,255852544~4194304,272629760~4194304,281018368~4194304,289406976~4194304,314572800~8388608,343932928~8388608,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.294 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,33554432~25165824,67108864~4194304,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~12582912,163577856~8388608,180355072~12582912,218103808~20971520,243269632~8388608,255852544~4194304,272629760~4194304,281018368~4194304,289406976~4194304,314572800~8388608,343932928~8388608,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.295 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,33554432~29360128,67108864~33554432,113246208~12582912,134217728~41943040,180355072~16777216,218103808~41943040,272629760~20971520,314572800~12582912,343932928~8388608,373293056~33554432] 2024-04-26T05:02:56.295 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 10 diff [0~16777216,20971520~4194304,33554432~25165824,67108864~4194304,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,272629760~12582912,289406976~4194304,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.295 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,33554432~25165824,67108864~4194304,75497472~4194304,83886080~12582912,113246208~8388608,134217728~8388608,146800640~25165824,180355072~12582912,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,272629760~12582912,289406976~4194304,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.298 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,33554432~29360128,67108864~33554432,113246208~12582912,134217728~41943040,180355072~16777216,201326592~12582912,218103808~75497472,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.298 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 11 diff [0~16777216,20971520~4194304,33554432~25165824,67108864~4194304,75497472~20971520,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,289406976~4194304,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.299 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,33554432~25165824,67108864~4194304,75497472~20971520,104857600~4194304,113246208~8388608,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,289406976~4194304,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.301 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,33554432~29360128,67108864~33554432,104857600~20971520,130023424~46137344,180355072~16777216,201326592~12582912,218103808~75497472,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.302 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 12 diff [0~16777216,20971520~4194304,33554432~25165824,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,289406976~4194304,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.302 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,33554432~25165824,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,289406976~4194304,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.305 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,33554432~67108864,104857600~20971520,130023424~46137344,180355072~16777216,201326592~12582912,218103808~75497472,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.305 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 13 diff [0~16777216,20971520~37748736,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,289406976~4194304,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.305 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~37748736,62914560~8388608,75497472~25165824,104857600~16777216,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,289406976~4194304,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.309 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~79691776,104857600~20971520,130023424~46137344,180355072~16777216,201326592~12582912,218103808~75497472,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.309 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 14 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.309 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.314 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.314 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 15 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.314 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.319 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~20971520,130023424~46137344,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.319 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 16 diff [0~58720256,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.319 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.323 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~100663296,104857600~71303168,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.323 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 17 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.324 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~20971520,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.330 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.330 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 18 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.330 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.336 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.336 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to 19 diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.336 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.343 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.343 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 6 diff [4194304~4194304,92274688~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~8388608,272629760~4194304,394264576~8388608] 2024-04-26T05:02:56.343 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~4194304,92274688~8388608,134217728~8388608,150994944~8388608,163577856~8388608,180355072~8388608,272629760~4194304,394264576~8388608] 2024-04-26T05:02:56.344 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,92274688~8388608,134217728~12582912,150994944~25165824,180355072~12582912,272629760~8388608,394264576~12582912] 2024-04-26T05:02:56.344 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 7 diff [4194304~4194304,20971520~4194304,50331648~8388608,75497472~4194304,83886080~16777216,134217728~8388608,150994944~8388608,163577856~8388608,180355072~8388608,230686720~8388608,243269632~8388608,272629760~4194304,314572800~8388608,394264576~8388608] 2024-04-26T05:02:56.344 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~4194304,20971520~4194304,50331648~8388608,75497472~4194304,83886080~16777216,134217728~8388608,150994944~8388608,163577856~8388608,180355072~8388608,230686720~8388608,243269632~8388608,272629760~4194304,314572800~8388608,394264576~8388608] 2024-04-26T05:02:56.345 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,20971520~8388608,50331648~12582912,75497472~25165824,134217728~12582912,150994944~25165824,180355072~12582912,230686720~25165824,272629760~8388608,314572800~12582912,394264576~12582912] 2024-04-26T05:02:56.345 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 8 diff [4194304~8388608,20971520~4194304,50331648~8388608,75497472~4194304,83886080~12582912,134217728~8388608,146800640~12582912,163577856~8388608,180355072~8388608,218103808~20971520,243269632~8388608,272629760~4194304,314572800~8388608,373293056~8388608,394264576~8388608] 2024-04-26T05:02:56.345 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,75497472~4194304,83886080~12582912,134217728~8388608,146800640~12582912,163577856~8388608,180355072~8388608,218103808~20971520,243269632~8388608,272629760~4194304,314572800~8388608,373293056~8388608,394264576~8388608] 2024-04-26T05:02:56.347 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,20971520~8388608,50331648~12582912,75497472~25165824,134217728~41943040,180355072~12582912,218103808~37748736,272629760~8388608,314572800~12582912,373293056~12582912,394264576~12582912] 2024-04-26T05:02:56.347 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 9 diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,134217728~8388608,146800640~12582912,163577856~8388608,180355072~8388608,218103808~20971520,243269632~8388608,255852544~4194304,272629760~4194304,314572800~8388608,343932928~8388608,373293056~8388608,394264576~12582912] 2024-04-26T05:02:56.347 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,134217728~8388608,146800640~12582912,163577856~8388608,180355072~8388608,218103808~20971520,243269632~8388608,255852544~4194304,272629760~4194304,314572800~8388608,343932928~8388608,373293056~8388608,394264576~12582912] 2024-04-26T05:02:56.349 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~20971520,67108864~33554432,134217728~41943040,180355072~12582912,218103808~41943040,272629760~8388608,314572800~12582912,343932928~8388608,373293056~12582912,394264576~12582912] 2024-04-26T05:02:56.349 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 10 diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,134217728~8388608,146800640~25165824,180355072~12582912,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,272629760~12582912,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,394264576~12582912] 2024-04-26T05:02:56.349 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,134217728~8388608,146800640~25165824,180355072~12582912,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,272629760~12582912,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,394264576~12582912] 2024-04-26T05:02:56.352 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~20971520,67108864~33554432,134217728~41943040,180355072~16777216,201326592~12582912,218103808~71303168,314572800~46137344,373293056~12582912,394264576~12582912] 2024-04-26T05:02:56.352 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 11 diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~20971520,104857600~4194304,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.352 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~20971520,104857600~4194304,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.355 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~20971520,67108864~33554432,104857600~8388608,130023424~46137344,180355072~16777216,201326592~12582912,218103808~71303168,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.355 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 12 diff [0~16777216,20971520~4194304,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.355 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.358 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~58720256,104857600~16777216,130023424~46137344,180355072~16777216,201326592~12582912,218103808~71303168,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.359 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 13 diff [0~16777216,20971520~16777216,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.359 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~16777216,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,218103808~50331648,272629760~12582912,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.362 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~16777216,41943040~58720256,104857600~16777216,130023424~46137344,180355072~16777216,201326592~12582912,218103808~71303168,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.362 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 14 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.362 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.366 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~46137344,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.367 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 15 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.367 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.372 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~46137344,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.372 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 16 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.372 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~37748736,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.377 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~71303168,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.378 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 17 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.378 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,272629760~16777216,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.383 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~16777216,201326592~92274688,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.383 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 18 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.384 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.390 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.390 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to 19 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.390 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.396 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.396 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 7 diff [20971520~4194304,50331648~8388608,75497472~4194304,83886080~8388608,163577856~4194304,230686720~8388608,243269632~8388608,314572800~8388608] 2024-04-26T05:02:56.396 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [20971520~4194304,50331648~8388608,75497472~4194304,83886080~8388608,163577856~4194304,230686720~8388608,243269632~8388608,314572800~8388608] 2024-04-26T05:02:56.397 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [20971520~8388608,50331648~12582912,75497472~16777216,163577856~8388608,230686720~25165824,314572800~12582912] 2024-04-26T05:02:56.397 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 8 diff [4194304~8388608,20971520~4194304,50331648~8388608,75497472~4194304,83886080~12582912,146800640~8388608,163577856~8388608,218103808~20971520,243269632~8388608,314572800~8388608,373293056~8388608,398458880~4194304] 2024-04-26T05:02:56.397 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,75497472~4194304,83886080~12582912,146800640~8388608,163577856~8388608,218103808~20971520,243269632~8388608,314572800~8388608,373293056~8388608,398458880~4194304] 2024-04-26T05:02:56.398 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,20971520~8388608,50331648~12582912,75497472~25165824,146800640~8388608,163577856~12582912,218103808~37748736,314572800~12582912,373293056~12582912,398458880~4194304] 2024-04-26T05:02:56.399 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 9 diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,146800640~8388608,163577856~8388608,184549376~4194304,218103808~20971520,243269632~8388608,255852544~4194304,314572800~8388608,343932928~8388608,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.399 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,146800640~8388608,163577856~8388608,184549376~4194304,218103808~20971520,243269632~8388608,255852544~4194304,314572800~8388608,343932928~8388608,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.401 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~20971520,67108864~33554432,146800640~8388608,163577856~12582912,184549376~8388608,218103808~41943040,314572800~12582912,343932928~8388608,373293056~12582912,398458880~8388608] 2024-04-26T05:02:56.401 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 10 diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.401 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~4194304,83886080~12582912,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~20971520,243269632~8388608,255852544~12582912,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.404 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~20971520,67108864~33554432,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~46137344,373293056~12582912,398458880~8388608] 2024-04-26T05:02:56.404 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 11 diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~20971520,104857600~4194304,130023424~8388608,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~50331648,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.404 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,67108864~4194304,75497472~20971520,104857600~4194304,130023424~8388608,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~50331648,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.406 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~20971520,67108864~33554432,104857600~8388608,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.407 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 12 diff [0~16777216,20971520~4194304,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~50331648,276824064~8388608,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.407 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~50331648,276824064~8388608,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.410 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~58720256,104857600~16777216,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~46137344,373293056~33554432] 2024-04-26T05:02:56.410 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 13 diff [0~16777216,20971520~16777216,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~50331648,276824064~8388608,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.410 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~16777216,41943040~16777216,62914560~8388608,75497472~25165824,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~50331648,276824064~8388608,314572800~20971520,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.413 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~16777216,41943040~58720256,104857600~16777216,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.414 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 14 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.414 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.418 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.418 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 15 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.418 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~20971520,130023424~12582912,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.421 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~29360128,163577856~12582912,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.422 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 16 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~37748736,146800640~12582912,163577856~8388608,180355072~12582912,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.422 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~25165824,104857600~37748736,146800640~12582912,163577856~8388608,180355072~12582912,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.426 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~54525952,163577856~12582912,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.426 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 17 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.426 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~12582912,201326592~8388608,213909504~54525952,276824064~12582912,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.430 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.430 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 18 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.430 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.435 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.435 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to 19 diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.435 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.440 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.440 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 8 diff [4194304~8388608,20971520~4194304,88080384~8388608,146800640~8388608,163577856~8388608,218103808~12582912,373293056~8388608,398458880~4194304] 2024-04-26T05:02:56.440 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,20971520~4194304,88080384~8388608,146800640~8388608,163577856~8388608,218103808~12582912,373293056~8388608,398458880~4194304] 2024-04-26T05:02:56.441 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,20971520~8388608,88080384~12582912,146800640~8388608,163577856~12582912,218103808~12582912,373293056~12582912,398458880~4194304] 2024-04-26T05:02:56.441 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 9 diff [0~16777216,20971520~4194304,41943040~8388608,67108864~4194304,88080384~8388608,146800640~8388608,163577856~8388608,184549376~4194304,218103808~12582912,255852544~4194304,343932928~8388608,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.441 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~8388608,67108864~4194304,88080384~8388608,146800640~8388608,163577856~8388608,184549376~4194304,218103808~12582912,255852544~4194304,343932928~8388608,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.443 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~12582912,67108864~8388608,88080384~12582912,146800640~8388608,163577856~12582912,184549376~8388608,218103808~12582912,255852544~4194304,343932928~8388608,373293056~12582912,398458880~8388608] 2024-04-26T05:02:56.443 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 10 diff [0~16777216,20971520~4194304,41943040~8388608,67108864~4194304,88080384~8388608,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~12582912,255852544~12582912,276824064~8388608,327155712~8388608,339738624~16777216,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.443 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~8388608,67108864~4194304,88080384~8388608,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~12582912,255852544~12582912,276824064~8388608,327155712~8388608,339738624~16777216,373293056~8388608,398458880~8388608] 2024-04-26T05:02:56.445 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~12582912,67108864~8388608,88080384~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~12582912,255852544~16777216,276824064~12582912,327155712~33554432,373293056~12582912,398458880~8388608] 2024-04-26T05:02:56.445 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 11 diff [0~16777216,20971520~4194304,41943040~8388608,67108864~4194304,75497472~8388608,88080384~8388608,104857600~4194304,130023424~8388608,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~12582912,239075328~8388608,251658240~16777216,276824064~8388608,314572800~4194304,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.445 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~8388608,67108864~4194304,75497472~8388608,88080384~8388608,104857600~4194304,130023424~8388608,146800640~8388608,163577856~8388608,184549376~8388608,201326592~8388608,218103808~12582912,239075328~8388608,251658240~16777216,276824064~8388608,314572800~4194304,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.447 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~12582912,67108864~33554432,104857600~8388608,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~12582912,239075328~33554432,276824064~12582912,314572800~8388608,327155712~33554432,373293056~33554432] 2024-04-26T05:02:56.447 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 12 diff [0~16777216,20971520~4194304,41943040~12582912,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~12582912,239075328~8388608,251658240~16777216,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.447 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~4194304,41943040~12582912,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~12582912,239075328~8388608,251658240~16777216,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.450 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~8388608,41943040~16777216,62914560~37748736,104857600~16777216,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~12582912,239075328~33554432,276824064~12582912,314572800~8388608,327155712~33554432,373293056~33554432] 2024-04-26T05:02:56.450 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 13 diff [0~16777216,20971520~12582912,41943040~16777216,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~16777216,239075328~29360128,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.450 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,20971520~12582912,41943040~16777216,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,218103808~16777216,239075328~29360128,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~8388608,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.453 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,20971520~16777216,41943040~58720256,104857600~16777216,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~8388608,327155712~33554432,373293056~41943040] 2024-04-26T05:02:56.453 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 14 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.453 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.457 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~12582912,146800640~12582912,163577856~12582912,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.457 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 15 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.457 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,146800640~12582912,163577856~8388608,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.462 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~29360128,163577856~12582912,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.462 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 16 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~12582912,163577856~8388608,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.462 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~12582912,163577856~8388608,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.466 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~54525952,163577856~12582912,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.466 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 17 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.466 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.470 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.470 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 18 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~20971520,239075328~29360128,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.470 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~20971520,239075328~29360128,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.476 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.476 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to 19 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.476 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.480 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.481 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 9 diff [0~8388608,12582912~4194304,41943040~8388608,67108864~4194304,167772160~4194304,184549376~4194304,218103808~8388608,255852544~4194304,343932928~8388608,398458880~8388608] 2024-04-26T05:02:56.481 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~8388608,12582912~4194304,41943040~8388608,67108864~4194304,167772160~4194304,184549376~4194304,218103808~8388608,255852544~4194304,343932928~8388608,398458880~8388608] 2024-04-26T05:02:56.482 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~8388608,12582912~4194304,41943040~12582912,67108864~8388608,167772160~4194304,184549376~8388608,218103808~8388608,255852544~4194304,343932928~8388608,398458880~8388608] 2024-04-26T05:02:56.482 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 10 diff [0~8388608,12582912~4194304,41943040~8388608,67108864~4194304,146800640~8388608,167772160~4194304,184549376~8388608,201326592~8388608,218103808~8388608,255852544~12582912,276824064~8388608,327155712~8388608,339738624~16777216,398458880~8388608] 2024-04-26T05:02:56.482 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~8388608,12582912~4194304,41943040~8388608,67108864~4194304,146800640~8388608,167772160~4194304,184549376~8388608,201326592~8388608,218103808~8388608,255852544~12582912,276824064~8388608,327155712~8388608,339738624~16777216,398458880~8388608] 2024-04-26T05:02:56.483 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~8388608,12582912~4194304,41943040~12582912,67108864~8388608,146800640~12582912,167772160~4194304,184549376~12582912,201326592~12582912,218103808~8388608,255852544~16777216,276824064~12582912,327155712~33554432,398458880~8388608] 2024-04-26T05:02:56.483 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 11 diff [0~16777216,41943040~8388608,67108864~4194304,75497472~8388608,88080384~4194304,104857600~4194304,130023424~8388608,146800640~8388608,167772160~4194304,184549376~8388608,201326592~8388608,218103808~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~4194304,327155712~8388608,339738624~16777216,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.483 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,41943040~8388608,67108864~4194304,75497472~8388608,88080384~4194304,104857600~4194304,130023424~8388608,146800640~8388608,167772160~4194304,184549376~8388608,201326592~8388608,218103808~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~4194304,327155712~8388608,339738624~16777216,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.485 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,41943040~12582912,67108864~25165824,104857600~8388608,130023424~12582912,146800640~12582912,167772160~4194304,184549376~12582912,201326592~12582912,218103808~8388608,239075328~33554432,276824064~12582912,314572800~8388608,327155712~33554432,385875968~20971520] 2024-04-26T05:02:56.485 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 12 diff [0~16777216,41943040~12582912,62914560~8388608,75497472~16777216,104857600~12582912,130023424~8388608,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,218103808~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.486 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,41943040~12582912,62914560~8388608,75497472~16777216,104857600~12582912,130023424~8388608,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,218103808~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,385875968~8388608,398458880~8388608] 2024-04-26T05:02:56.488 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,41943040~16777216,62914560~33554432,104857600~16777216,130023424~12582912,146800640~12582912,167772160~4194304,184549376~12582912,201326592~12582912,218103808~8388608,239075328~33554432,276824064~12582912,314572800~8388608,327155712~33554432,385875968~20971520] 2024-04-26T05:02:56.488 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 13 diff [0~16777216,25165824~12582912,41943040~16777216,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,218103808~16777216,239075328~29360128,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~4194304,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.488 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~16777216,25165824~12582912,41943040~16777216,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,218103808~16777216,239075328~29360128,276824064~8388608,314572800~8388608,327155712~8388608,339738624~16777216,373293056~4194304,385875968~8388608,398458880~12582912] 2024-04-26T05:02:56.491 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~16777216,25165824~12582912,41943040~58720256,104857600~16777216,130023424~12582912,146800640~12582912,167772160~4194304,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~8388608,327155712~33554432,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.491 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 14 diff [0~20971520,25165824~12582912,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~4194304,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.491 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~20971520,25165824~12582912,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~4194304,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.495 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~20971520,25165824~12582912,41943040~58720256,104857600~20971520,130023424~12582912,146800640~12582912,167772160~4194304,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.495 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 15 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.495 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,146800640~12582912,167772160~4194304,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.498 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~20971520,130023424~29360128,167772160~4194304,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.498 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 16 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~12582912,167772160~4194304,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.498 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~12582912,167772160~4194304,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,398458880~16777216] 2024-04-26T05:02:56.503 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~58720256,104857600~54525952,167772160~4194304,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~8388608,385875968~29360128] 2024-04-26T05:02:56.503 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 17 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.503 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.507 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~130023424,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.507 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 18 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~20971520,239075328~29360128,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.507 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~20971520,239075328~29360128,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.512 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~130023424,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.512 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to 19 diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.512 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.517 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.517 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 10 diff [146800640~8388608,184549376~8388608,201326592~8388608,260046848~8388608,276824064~8388608,327155712~8388608,339738624~4194304,352321536~4194304] 2024-04-26T05:02:56.517 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [146800640~8388608,184549376~8388608,201326592~8388608,260046848~8388608,276824064~8388608,327155712~8388608,339738624~4194304,352321536~4194304] 2024-04-26T05:02:56.518 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [146800640~12582912,184549376~12582912,201326592~12582912,260046848~12582912,276824064~12582912,327155712~20971520,352321536~8388608] 2024-04-26T05:02:56.518 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 11 diff [8388608~8388608,75497472~8388608,88080384~4194304,104857600~4194304,130023424~8388608,146800640~8388608,184549376~8388608,201326592~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~4194304,327155712~8388608,339738624~4194304,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.518 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,75497472~8388608,88080384~4194304,104857600~4194304,130023424~8388608,146800640~8388608,184549376~8388608,201326592~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~4194304,327155712~8388608,339738624~4194304,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.520 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,75497472~16777216,104857600~8388608,130023424~12582912,146800640~12582912,184549376~12582912,201326592~12582912,239075328~33554432,276824064~12582912,314572800~8388608,327155712~20971520,352321536~8388608,385875968~12582912] 2024-04-26T05:02:56.520 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 12 diff [8388608~8388608,50331648~8388608,62914560~8388608,75497472~16777216,104857600~12582912,130023424~8388608,146800640~12582912,184549376~8388608,201326592~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~8388608,327155712~8388608,339738624~4194304,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.520 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,50331648~8388608,62914560~8388608,75497472~16777216,104857600~12582912,130023424~8388608,146800640~12582912,184549376~8388608,201326592~8388608,239075328~8388608,251658240~16777216,276824064~8388608,314572800~8388608,327155712~8388608,339738624~4194304,352321536~4194304,385875968~8388608] 2024-04-26T05:02:56.522 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,50331648~8388608,62914560~8388608,75497472~20971520,104857600~16777216,130023424~12582912,146800640~12582912,184549376~12582912,201326592~12582912,239075328~33554432,276824064~12582912,314572800~8388608,327155712~20971520,352321536~8388608,385875968~12582912] 2024-04-26T05:02:56.522 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 13 diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,184549376~8388608,201326592~8388608,218103808~16777216,239075328~29360128,276824064~8388608,314572800~8388608,327155712~8388608,339738624~4194304,348127232~8388608,373293056~4194304,385875968~8388608,406847488~4194304] 2024-04-26T05:02:56.522 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,146800640~12582912,184549376~8388608,201326592~8388608,218103808~16777216,239075328~29360128,276824064~8388608,314572800~8388608,327155712~8388608,339738624~4194304,348127232~8388608,373293056~4194304,385875968~8388608,406847488~4194304] 2024-04-26T05:02:56.525 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~12582912,50331648~20971520,75497472~25165824,104857600~16777216,130023424~12582912,146800640~12582912,184549376~12582912,201326592~12582912,218103808~54525952,276824064~12582912,314572800~8388608,327155712~33554432,373293056~8388608,385875968~12582912,406847488~8388608] 2024-04-26T05:02:56.525 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 14 diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,146800640~12582912,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~29360128,348127232~8388608,373293056~4194304,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.525 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,146800640~12582912,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~29360128,348127232~8388608,373293056~4194304,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.527 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~12582912,25165824~12582912,50331648~20971520,75497472~25165824,104857600~20971520,130023424~12582912,146800640~12582912,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~8388608,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.527 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 15 diff [4194304~12582912,20971520~12582912,46137344~12582912,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,146800640~12582912,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.527 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~12582912,20971520~12582912,46137344~12582912,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,146800640~12582912,184549376~8388608,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.531 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,46137344~25165824,75497472~25165824,104857600~20971520,130023424~29360128,184549376~12582912,201326592~71303168,276824064~16777216,314572800~46137344,373293056~8388608,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.531 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 16 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~12582912,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.531 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~12582912,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.535 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~25165824,104857600~54525952,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~8388608,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.535 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 17 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~20971520,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.535 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~20971520,180355072~12582912,201326592~8388608,213909504~20971520,239075328~29360128,276824064~12582912,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.539 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~92274688,180355072~16777216,201326592~71303168,276824064~16777216,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:02:56.539 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 18 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~20971520,180355072~29360128,213909504~20971520,239075328~29360128,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.539 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~20971520,180355072~29360128,213909504~20971520,239075328~29360128,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.544 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~92274688,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:02:56.544 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to 19 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.544 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.550 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:02:56.550 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 11 diff [8388608~8388608,75497472~8388608,88080384~4194304,104857600~4194304,130023424~8388608,239075328~8388608,251658240~8388608,314572800~4194304,385875968~8388608] 2024-04-26T05:02:56.550 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,75497472~8388608,88080384~4194304,104857600~4194304,130023424~8388608,239075328~8388608,251658240~8388608,314572800~4194304,385875968~8388608] 2024-04-26T05:02:56.551 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,75497472~16777216,104857600~8388608,130023424~12582912,239075328~20971520,314572800~8388608,385875968~12582912] 2024-04-26T05:02:56.551 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 12 diff [8388608~8388608,50331648~8388608,62914560~8388608,75497472~16777216,104857600~12582912,130023424~8388608,155189248~4194304,239075328~8388608,251658240~12582912,314572800~8388608,385875968~8388608] 2024-04-26T05:02:56.551 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,50331648~8388608,62914560~8388608,75497472~16777216,104857600~12582912,130023424~8388608,155189248~4194304,239075328~8388608,251658240~12582912,314572800~8388608,385875968~8388608] 2024-04-26T05:02:56.553 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,50331648~8388608,62914560~8388608,75497472~20971520,104857600~16777216,130023424~12582912,155189248~4194304,239075328~25165824,314572800~8388608,385875968~12582912] 2024-04-26T05:02:56.553 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 13 diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,155189248~4194304,218103808~16777216,239075328~25165824,314572800~8388608,348127232~4194304,373293056~4194304,385875968~8388608,406847488~4194304] 2024-04-26T05:02:56.553 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~12582912,130023424~8388608,155189248~4194304,218103808~16777216,239075328~25165824,314572800~8388608,348127232~4194304,373293056~4194304,385875968~8388608,406847488~4194304] 2024-04-26T05:02:56.555 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~12582912,50331648~20971520,75497472~25165824,104857600~16777216,130023424~12582912,155189248~4194304,218103808~46137344,314572800~8388608,348127232~4194304,373293056~8388608,385875968~12582912,406847488~8388608] 2024-04-26T05:02:56.555 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 14 diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,155189248~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~4194304,348127232~4194304,373293056~4194304,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.555 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~8388608,25165824~12582912,50331648~8388608,62914560~8388608,75497472~20971520,104857600~20971520,130023424~8388608,155189248~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~4194304,348127232~4194304,373293056~4194304,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.557 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~12582912,25165824~12582912,50331648~20971520,75497472~25165824,104857600~20971520,130023424~12582912,155189248~4194304,213909504~50331648,285212672~8388608,314572800~12582912,335544320~8388608,348127232~4194304,373293056~8388608,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.557 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 15 diff [4194304~12582912,20971520~12582912,46137344~12582912,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,150994944~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.558 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~12582912,20971520~12582912,46137344~12582912,62914560~8388608,75497472~20971520,104857600~20971520,130023424~12582912,150994944~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.560 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,46137344~25165824,75497472~25165824,104857600~20971520,130023424~16777216,150994944~8388608,213909504~50331648,285212672~8388608,314572800~12582912,335544320~16777216,373293056~8388608,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.560 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 16 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~8388608,180355072~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.560 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,104857600~37748736,146800640~8388608,180355072~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,385875968~8388608,402653184~12582912] 2024-04-26T05:02:56.564 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~25165824,104857600~54525952,180355072~4194304,213909504~50331648,285212672~8388608,314572800~12582912,335544320~16777216,373293056~8388608,385875968~12582912,402653184~12582912] 2024-04-26T05:02:56.564 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 17 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,180355072~8388608,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.564 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,180355072~8388608,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.567 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~92274688,180355072~8388608,213909504~50331648,276824064~16777216,314572800~12582912,335544320~16777216,373293056~25165824,402653184~12582912] 2024-04-26T05:02:56.567 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 18 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,180355072~8388608,192937984~12582912,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.567 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,180355072~8388608,192937984~12582912,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.571 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~92274688,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~16777216,373293056~25165824,402653184~12582912] 2024-04-26T05:02:56.571 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to 19 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.571 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.575 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:02:56.575 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 12 diff [50331648~8388608,62914560~8388608,83886080~12582912,109051904~8388608,155189248~4194304,239075328~8388608,255852544~8388608,314572800~4194304] 2024-04-26T05:02:56.575 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [50331648~8388608,62914560~8388608,83886080~12582912,109051904~8388608,155189248~4194304,239075328~8388608,255852544~8388608,314572800~4194304] 2024-04-26T05:02:56.577 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [50331648~8388608,62914560~8388608,83886080~12582912,109051904~12582912,155189248~4194304,239075328~8388608,255852544~8388608,314572800~4194304] 2024-04-26T05:02:56.577 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 13 diff [25165824~12582912,50331648~8388608,62914560~8388608,83886080~12582912,109051904~8388608,155189248~4194304,218103808~16777216,239075328~25165824,314572800~4194304,348127232~4194304,373293056~4194304,406847488~4194304] 2024-04-26T05:02:56.577 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [25165824~12582912,50331648~8388608,62914560~8388608,83886080~12582912,109051904~8388608,155189248~4194304,218103808~16777216,239075328~25165824,314572800~4194304,348127232~4194304,373293056~4194304,406847488~4194304] 2024-04-26T05:02:56.578 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [25165824~12582912,50331648~20971520,83886080~16777216,109051904~12582912,155189248~4194304,218103808~46137344,314572800~4194304,348127232~4194304,373293056~8388608,406847488~8388608] 2024-04-26T05:02:56.578 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 14 diff [12582912~4194304,25165824~12582912,50331648~8388608,62914560~8388608,83886080~12582912,109051904~16777216,155189248~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~4194304,348127232~4194304,373293056~4194304,402653184~12582912] 2024-04-26T05:02:56.579 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12582912~4194304,25165824~12582912,50331648~8388608,62914560~8388608,83886080~12582912,109051904~16777216,155189248~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~4194304,348127232~4194304,373293056~4194304,402653184~12582912] 2024-04-26T05:02:56.581 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12582912~8388608,25165824~12582912,50331648~20971520,83886080~16777216,109051904~16777216,155189248~4194304,213909504~50331648,285212672~8388608,314572800~12582912,335544320~8388608,348127232~4194304,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.581 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 15 diff [4194304~12582912,20971520~12582912,46137344~12582912,62914560~8388608,83886080~12582912,109051904~16777216,138412032~4194304,150994944~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.581 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~12582912,20971520~12582912,46137344~12582912,62914560~8388608,83886080~12582912,109051904~16777216,138412032~4194304,150994944~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.583 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,46137344~25165824,83886080~16777216,109051904~16777216,138412032~8388608,150994944~8388608,213909504~50331648,285212672~8388608,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.584 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 16 diff [4194304~29360128,41943040~16777216,62914560~8388608,79691776~16777216,109051904~20971520,138412032~4194304,146800640~8388608,180355072~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.584 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,79691776~16777216,109051904~20971520,138412032~4194304,146800640~8388608,180355072~4194304,213909504~20971520,239075328~25165824,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.587 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,79691776~20971520,109051904~20971520,138412032~20971520,180355072~4194304,213909504~50331648,285212672~8388608,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.587 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 17 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,180355072~8388608,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.587 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,180355072~8388608,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.589 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~29360128,180355072~8388608,213909504~50331648,276824064~16777216,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.590 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 18 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.590 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,213909504~20971520,239075328~25165824,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.594 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~29360128,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.594 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to 19 diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.594 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.597 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.597 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 13 diff [25165824~12582912,54525952~4194304,92274688~4194304,218103808~16777216,247463936~8388608,348127232~4194304,373293056~4194304,406847488~4194304] 2024-04-26T05:02:56.597 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [25165824~12582912,54525952~4194304,92274688~4194304,218103808~16777216,247463936~8388608,348127232~4194304,373293056~4194304,406847488~4194304] 2024-04-26T05:02:56.598 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [25165824~12582912,54525952~8388608,92274688~8388608,218103808~20971520,247463936~12582912,348127232~4194304,373293056~8388608,406847488~8388608] 2024-04-26T05:02:56.599 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 14 diff [12582912~4194304,25165824~12582912,54525952~4194304,92274688~4194304,117440512~8388608,213909504~20971520,247463936~8388608,285212672~4194304,318767104~8388608,335544320~4194304,348127232~4194304,373293056~4194304,402653184~12582912] 2024-04-26T05:02:56.599 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12582912~4194304,25165824~12582912,54525952~4194304,92274688~4194304,117440512~8388608,213909504~20971520,247463936~8388608,285212672~4194304,318767104~8388608,335544320~4194304,348127232~4194304,373293056~4194304,402653184~12582912] 2024-04-26T05:02:56.600 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12582912~8388608,25165824~12582912,54525952~8388608,92274688~8388608,117440512~8388608,213909504~25165824,247463936~12582912,285212672~8388608,318767104~8388608,335544320~8388608,348127232~4194304,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.600 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 15 diff [4194304~12582912,20971520~12582912,46137344~12582912,83886080~12582912,117440512~8388608,138412032~4194304,150994944~4194304,213909504~20971520,247463936~8388608,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.600 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~12582912,20971520~12582912,46137344~12582912,83886080~12582912,117440512~8388608,138412032~4194304,150994944~4194304,213909504~20971520,247463936~8388608,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.602 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,46137344~16777216,83886080~16777216,117440512~8388608,138412032~8388608,150994944~8388608,213909504~25165824,247463936~12582912,285212672~8388608,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.602 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 16 diff [4194304~29360128,41943040~16777216,79691776~16777216,113246208~16777216,138412032~4194304,146800640~8388608,180355072~4194304,213909504~20971520,247463936~8388608,260046848~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.602 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,79691776~16777216,113246208~16777216,138412032~4194304,146800640~8388608,180355072~4194304,213909504~20971520,247463936~8388608,260046848~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.604 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,79691776~20971520,113246208~16777216,138412032~20971520,180355072~4194304,213909504~25165824,247463936~16777216,285212672~8388608,314572800~12582912,335544320~16777216,373293056~8388608,402653184~12582912] 2024-04-26T05:02:56.604 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 17 diff [4194304~29360128,41943040~16777216,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,213909504~20971520,247463936~8388608,260046848~4194304,276824064~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.604 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,213909504~20971520,247463936~8388608,260046848~4194304,276824064~4194304,285212672~4194304,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.606 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,75497472~29360128,113246208~16777216,138412032~29360128,180355072~8388608,213909504~25165824,247463936~16777216,276824064~16777216,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.606 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 18 diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,213909504~20971520,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.606 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,213909504~20971520,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.609 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~29360128,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.609 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to 19 diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.609 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.613 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:56.613 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 14 diff [12582912~4194304,117440512~8388608,213909504~8388608,285212672~4194304,318767104~8388608,335544320~4194304,402653184~12582912] 2024-04-26T05:02:56.613 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [12582912~4194304,117440512~8388608,213909504~8388608,285212672~4194304,318767104~8388608,335544320~4194304,402653184~12582912] 2024-04-26T05:02:56.614 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [12582912~8388608,117440512~8388608,213909504~12582912,285212672~8388608,318767104~8388608,335544320~8388608,402653184~12582912] 2024-04-26T05:02:56.614 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 15 diff [4194304~12582912,20971520~4194304,46137344~8388608,83886080~8388608,117440512~8388608,138412032~4194304,150994944~4194304,213909504~8388608,285212672~4194304,314572800~12582912,335544320~12582912,377487360~4194304,402653184~12582912] 2024-04-26T05:02:56.614 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~12582912,20971520~4194304,46137344~8388608,83886080~8388608,117440512~8388608,138412032~4194304,150994944~4194304,213909504~8388608,285212672~4194304,314572800~12582912,335544320~12582912,377487360~4194304,402653184~12582912] 2024-04-26T05:02:56.616 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~25165824,46137344~12582912,83886080~12582912,117440512~8388608,138412032~8388608,150994944~8388608,213909504~12582912,285212672~8388608,314572800~12582912,335544320~16777216,377487360~4194304,402653184~12582912] 2024-04-26T05:02:56.616 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 16 diff [4194304~20971520,41943040~12582912,79691776~12582912,113246208~16777216,138412032~4194304,146800640~8388608,180355072~4194304,213909504~8388608,260046848~4194304,285212672~4194304,314572800~12582912,335544320~12582912,377487360~4194304,402653184~12582912] 2024-04-26T05:02:56.616 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~20971520,41943040~12582912,79691776~12582912,113246208~16777216,138412032~4194304,146800640~8388608,180355072~4194304,213909504~8388608,260046848~4194304,285212672~4194304,314572800~12582912,335544320~12582912,377487360~4194304,402653184~12582912] 2024-04-26T05:02:56.618 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~25165824,41943040~16777216,79691776~16777216,113246208~16777216,138412032~20971520,180355072~4194304,213909504~12582912,260046848~4194304,285212672~8388608,314572800~12582912,335544320~16777216,377487360~4194304,402653184~12582912] 2024-04-26T05:02:56.618 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 17 diff [4194304~20971520,41943040~12582912,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,213909504~8388608,251658240~4194304,260046848~4194304,276824064~4194304,285212672~4194304,314572800~12582912,335544320~12582912,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.618 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~20971520,41943040~12582912,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,213909504~8388608,251658240~4194304,260046848~4194304,276824064~4194304,285212672~4194304,314572800~12582912,335544320~12582912,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.620 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~25165824,41943040~16777216,75497472~20971520,100663296~4194304,113246208~16777216,138412032~29360128,180355072~8388608,213909504~12582912,251658240~12582912,276824064~16777216,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.620 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 18 diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,213909504~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~12582912,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.620 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,213909504~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~12582912,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.622 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~29360128,180355072~8388608,192937984~12582912,213909504~12582912,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.622 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to 19 diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.622 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.625 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~12582912,230686720~8388608,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,377487360~8388608,402653184~12582912] 2024-04-26T05:02:56.625 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 15 diff [4194304~8388608,20971520~4194304,46137344~8388608,83886080~8388608,138412032~4194304,150994944~4194304,314572800~8388608,339738624~8388608,377487360~4194304] 2024-04-26T05:02:56.625 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,20971520~4194304,46137344~8388608,83886080~8388608,138412032~4194304,150994944~4194304,314572800~8388608,339738624~8388608,377487360~4194304] 2024-04-26T05:02:56.626 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,20971520~8388608,46137344~12582912,83886080~12582912,138412032~8388608,150994944~8388608,314572800~8388608,339738624~12582912,377487360~4194304] 2024-04-26T05:02:56.626 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 16 diff [4194304~8388608,16777216~8388608,41943040~12582912,79691776~12582912,113246208~16777216,138412032~4194304,146800640~8388608,180355072~4194304,260046848~4194304,314572800~12582912,339738624~8388608,377487360~4194304] 2024-04-26T05:02:56.626 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~8388608,41943040~12582912,79691776~12582912,113246208~16777216,138412032~4194304,146800640~8388608,180355072~4194304,260046848~4194304,314572800~12582912,339738624~8388608,377487360~4194304] 2024-04-26T05:02:56.628 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,79691776~16777216,113246208~16777216,138412032~20971520,180355072~4194304,260046848~4194304,314572800~12582912,339738624~12582912,377487360~4194304] 2024-04-26T05:02:56.628 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 17 diff [4194304~8388608,16777216~8388608,41943040~12582912,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,251658240~4194304,260046848~4194304,276824064~4194304,289406976~4194304,314572800~12582912,339738624~8388608,377487360~8388608] 2024-04-26T05:02:56.628 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~8388608,41943040~12582912,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,251658240~4194304,260046848~4194304,276824064~4194304,289406976~4194304,314572800~12582912,339738624~8388608,377487360~8388608] 2024-04-26T05:02:56.630 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,75497472~20971520,100663296~4194304,113246208~16777216,138412032~29360128,180355072~8388608,251658240~12582912,276824064~8388608,289406976~4194304,314572800~12582912,339738624~12582912,377487360~8388608] 2024-04-26T05:02:56.630 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 18 diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~8388608,377487360~8388608] 2024-04-26T05:02:56.630 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,180355072~8388608,192937984~12582912,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~8388608,377487360~8388608] 2024-04-26T05:02:56.632 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~8388608,16777216~16777216,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~29360128,180355072~8388608,192937984~12582912,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608] 2024-04-26T05:02:56.632 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to 19 diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:56.632 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:56.635 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,314572800~12582912,339738624~16777216,377487360~8388608,402653184~8388608] 2024-04-26T05:02:56.635 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 16 diff [16777216~4194304,41943040~4194304,79691776~12582912,113246208~16777216,146800640~8388608,180355072~4194304,260046848~4194304,318767104~8388608] 2024-04-26T05:02:56.635 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [16777216~4194304,41943040~4194304,79691776~12582912,113246208~16777216,146800640~8388608,180355072~4194304,260046848~4194304,318767104~8388608] 2024-04-26T05:02:56.636 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [16777216~4194304,41943040~8388608,79691776~12582912,113246208~16777216,146800640~8388608,180355072~4194304,260046848~4194304,318767104~8388608] 2024-04-26T05:02:56.636 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 17 diff [16777216~4194304,41943040~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,180355072~8388608,251658240~4194304,260046848~4194304,276824064~4194304,289406976~4194304,318767104~8388608,381681664~4194304] 2024-04-26T05:02:56.636 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [16777216~4194304,41943040~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,180355072~8388608,251658240~4194304,260046848~4194304,276824064~4194304,289406976~4194304,318767104~8388608,381681664~4194304] 2024-04-26T05:02:56.637 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [16777216~4194304,41943040~8388608,75497472~16777216,100663296~4194304,113246208~16777216,146800640~20971520,180355072~8388608,251658240~12582912,276824064~8388608,289406976~4194304,318767104~8388608,381681664~4194304] 2024-04-26T05:02:56.637 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 18 diff [16777216~4194304,25165824~8388608,41943040~4194304,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,180355072~8388608,192937984~12582912,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,381681664~4194304] 2024-04-26T05:02:56.637 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [16777216~4194304,25165824~8388608,41943040~4194304,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,180355072~8388608,192937984~12582912,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,381681664~4194304] 2024-04-26T05:02:56.640 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [16777216~4194304,25165824~8388608,41943040~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~20971520,180355072~8388608,192937984~12582912,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,318767104~8388608,381681664~4194304] 2024-04-26T05:02:56.640 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to 19 diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:56.640 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:56.642 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~12582912,25165824~8388608,41943040~16777216,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~29360128,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:56.642 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to 17 diff [75497472~8388608,100663296~4194304,121634816~4194304,155189248~8388608,184549376~4194304,251658240~4194304,276824064~4194304,289406976~4194304,322961408~4194304,381681664~4194304] 2024-04-26T05:02:56.642 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [75497472~8388608,100663296~4194304,121634816~4194304,155189248~8388608,184549376~4194304,251658240~4194304,276824064~4194304,289406976~4194304,322961408~4194304,381681664~4194304] 2024-04-26T05:02:56.643 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [75497472~12582912,100663296~4194304,121634816~4194304,155189248~12582912,184549376~4194304,251658240~8388608,276824064~8388608,289406976~4194304,322961408~4194304,381681664~4194304] 2024-04-26T05:02:56.643 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to 18 diff [25165824~8388608,41943040~4194304,67108864~4194304,75497472~8388608,100663296~4194304,121634816~4194304,146800640~4194304,155189248~8388608,184549376~4194304,192937984~12582912,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,381681664~4194304] 2024-04-26T05:02:56.643 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [25165824~8388608,41943040~4194304,67108864~4194304,75497472~8388608,100663296~4194304,121634816~4194304,146800640~4194304,155189248~8388608,184549376~4194304,192937984~12582912,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,381681664~4194304] 2024-04-26T05:02:56.644 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [25165824~8388608,41943040~8388608,67108864~4194304,75497472~12582912,100663296~4194304,121634816~8388608,146800640~4194304,155189248~12582912,184549376~4194304,192937984~12582912,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,322961408~4194304,381681664~4194304] 2024-04-26T05:02:56.645 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to 19 diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~8388608,100663296~4194304,121634816~8388608,146800640~4194304,155189248~8388608,167772160~4194304,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:56.645 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~8388608,100663296~4194304,121634816~8388608,146800640~4194304,155189248~8388608,167772160~4194304,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:56.646 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~8388608,41943040~16777216,67108864~4194304,75497472~12582912,100663296~4194304,121634816~8388608,146800640~4194304,155189248~20971520,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:56.646 INFO:tasks.workunit.client.0.smithi005.stdout:from 17 to 18 diff [25165824~8388608,41943040~4194304,67108864~4194304,121634816~4194304,146800640~4194304,192937984~12582912,251658240~8388608,297795584~8388608,322961408~4194304] 2024-04-26T05:02:56.646 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [25165824~8388608,41943040~4194304,67108864~4194304,121634816~4194304,146800640~4194304,192937984~12582912,251658240~8388608,297795584~8388608,322961408~4194304] 2024-04-26T05:02:56.647 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [25165824~8388608,41943040~8388608,67108864~4194304,121634816~8388608,146800640~4194304,192937984~12582912,251658240~12582912,297795584~8388608,322961408~4194304] 2024-04-26T05:02:56.647 INFO:tasks.workunit.client.0.smithi005.stdout:from 17 to 19 diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,146800640~4194304,167772160~4194304,192937984~12582912,230686720~8388608,251658240~12582912,297795584~8388608,322961408~4194304,348127232~8388608,402653184~8388608] 2024-04-26T05:02:56.648 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,146800640~4194304,167772160~4194304,192937984~12582912,230686720~8388608,251658240~12582912,297795584~8388608,322961408~4194304,348127232~8388608,402653184~8388608] 2024-04-26T05:02:56.649 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~8388608,41943040~16777216,67108864~4194304,100663296~4194304,121634816~8388608,146800640~4194304,167772160~8388608,192937984~12582912,230686720~8388608,251658240~12582912,297795584~8388608,322961408~4194304,348127232~8388608,402653184~8388608] 2024-04-26T05:02:56.649 INFO:tasks.workunit.client.0.smithi005.stdout:from 18 to 19 diff [8388608~4194304,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,167772160~4194304,230686720~8388608,255852544~8388608,348127232~8388608,402653184~8388608] 2024-04-26T05:02:56.649 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,167772160~4194304,230686720~8388608,255852544~8388608,348127232~8388608,402653184~8388608] 2024-04-26T05:02:56.650 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,167772160~8388608,230686720~8388608,255852544~8388608,348127232~8388608,402653184~8388608] 2024-04-26T05:02:56.930 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:56.930 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:56.951 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:56.951 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.951 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.959 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.959 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.960 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.966 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:56.966 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.966 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.973 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:56.973 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.973 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.979 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.979 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.979 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.985 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.985 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.986 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.990 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.990 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.990 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.995 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.995 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.995 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:56.999 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:56.999 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:56.999 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:57.004 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:02:57.004 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:57.004 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:57.008 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:02:57.008 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:57.008 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:57.011 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:57.011 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:57.011 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:57.015 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:57.015 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:57.015 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:57.017 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~12582912,230686720~8388608,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,377487360~8388608,402653184~12582912] 2024-04-26T05:02:57.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:57.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:57.019 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,314572800~12582912,339738624~16777216,377487360~8388608,402653184~8388608] 2024-04-26T05:02:57.019 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to HEAD diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:57.020 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:57.021 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~12582912,25165824~8388608,41943040~16777216,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~29360128,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:57.021 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to HEAD diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~8388608,100663296~4194304,121634816~8388608,146800640~4194304,155189248~8388608,167772160~4194304,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:57.021 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~8388608,100663296~4194304,121634816~8388608,146800640~4194304,155189248~8388608,167772160~4194304,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:57.022 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~8388608,41943040~16777216,67108864~4194304,75497472~12582912,100663296~4194304,121634816~8388608,146800640~4194304,155189248~20971520,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:57.023 INFO:tasks.workunit.client.0.smithi005.stdout:from 17 to HEAD diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,146800640~4194304,167772160~4194304,192937984~12582912,230686720~8388608,251658240~12582912,297795584~8388608,322961408~4194304,348127232~8388608,402653184~8388608] 2024-04-26T05:02:57.023 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,100663296~4194304,121634816~8388608,146800640~4194304,167772160~4194304,192937984~12582912,230686720~8388608,251658240~12582912,297795584~8388608,322961408~4194304,348127232~8388608,402653184~8388608] 2024-04-26T05:02:57.024 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~8388608,41943040~16777216,67108864~4194304,100663296~4194304,121634816~8388608,146800640~4194304,167772160~8388608,192937984~12582912,230686720~8388608,251658240~12582912,297795584~8388608,322961408~4194304,348127232~8388608,402653184~8388608] 2024-04-26T05:02:57.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:56 smithi005 ceph-mon[91801]: pgmap v1096: 105 pgs: 105 active+clean; 776 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 116 KiB/s rd, 32 MiB/s wr, 170 op/s 2024-04-26T05:02:57.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:56 smithi005 ceph-mon[91801]: osdmap e938: 8 total, 8 up, 8 in 2024-04-26T05:02:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:56 smithi005 ceph-mon[95809]: pgmap v1096: 105 pgs: 105 active+clean; 776 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 116 KiB/s rd, 32 MiB/s wr, 170 op/s 2024-04-26T05:02:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:56 smithi005 ceph-mon[95809]: osdmap e938: 8 total, 8 up, 8 in 2024-04-26T05:02:57.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:56 smithi111 ceph-mon[86360]: pgmap v1096: 105 pgs: 105 active+clean; 776 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 116 KiB/s rd, 32 MiB/s wr, 170 op/s 2024-04-26T05:02:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:56 smithi111 ceph-mon[86360]: osdmap e938: 8 total, 8 up, 8 in 2024-04-26T05:02:57.947 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:57.948 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:57.965 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:57.965 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:57.965 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:57.973 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:57.973 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.974 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.981 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:57.981 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.981 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.988 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:57.988 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.988 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.994 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:57.994 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:57.994 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.000 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:58.000 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.000 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.006 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:58.006 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.006 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.011 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:58.011 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.011 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.015 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:58.015 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:58.015 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:58.019 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:02:58.019 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:58.019 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:58.023 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:02:58.023 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:58.023 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:58.026 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:58.027 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:58.027 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:58.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:58.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:58.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:58.032 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~12582912,230686720~8388608,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,377487360~8388608,402653184~12582912] 2024-04-26T05:02:58.032 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:58.032 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:58.034 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,314572800~12582912,339738624~16777216,377487360~8388608,402653184~8388608] 2024-04-26T05:02:58.034 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to HEAD diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:58.034 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:58.036 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~12582912,25165824~8388608,41943040~16777216,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~29360128,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:58.036 INFO:tasks.workunit.client.0.smithi005.stdout:from 16 to HEAD diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~8388608,100663296~4194304,121634816~8388608,146800640~4194304,155189248~8388608,167772160~4194304,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:58.036 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~8388608,100663296~4194304,121634816~8388608,146800640~4194304,155189248~8388608,167772160~4194304,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:58.037 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~8388608,25165824~8388608,41943040~16777216,67108864~4194304,75497472~12582912,100663296~4194304,121634816~8388608,146800640~4194304,155189248~20971520,184549376~4194304,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,322961408~4194304,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:58.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:57 smithi005 ceph-mon[91801]: osdmap e939: 8 total, 8 up, 8 in 2024-04-26T05:02:58.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:57 smithi005 ceph-mon[95809]: osdmap e939: 8 total, 8 up, 8 in 2024-04-26T05:02:58.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:57 smithi111 ceph-mon[86360]: osdmap e939: 8 total, 8 up, 8 in 2024-04-26T05:02:58.954 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:58.955 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:58.974 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:58.974 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:58.974 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:58.981 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:58.981 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.981 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.988 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:58.988 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.988 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.995 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:58.995 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:58.995 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.001 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:59.001 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.001 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.007 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:59.007 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.007 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.012 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:59.012 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.012 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.017 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:59.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.022 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:02:59.022 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:59.022 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:02:59.026 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:02:59.026 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:59.026 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:02:59.030 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:02:59.030 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:59.030 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:59.033 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:59.033 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:59.033 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:02:59.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:02:59.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:59.035 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:02:59.037 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~12582912,230686720~8388608,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,377487360~8388608,402653184~12582912] 2024-04-26T05:02:59.038 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:59.038 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:02:59.040 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,314572800~12582912,339738624~16777216,377487360~8388608,402653184~8388608] 2024-04-26T05:02:59.040 INFO:tasks.workunit.client.0.smithi005.stdout:from 15 to HEAD diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:59.040 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [8388608~4194304,16777216~4194304,25165824~8388608,41943040~4194304,50331648~8388608,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:59.041 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [8388608~12582912,25165824~8388608,41943040~16777216,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,146800640~29360128,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,318767104~8388608,348127232~8388608,381681664~4194304,402653184~8388608] 2024-04-26T05:02:59.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:58 smithi005 ceph-mon[95809]: pgmap v1099: 105 pgs: 105 active+clean; 901 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 510 KiB/s rd, 32 MiB/s wr, 904 op/s 2024-04-26T05:02:59.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:58 smithi005 ceph-mon[95809]: osdmap e940: 8 total, 8 up, 8 in 2024-04-26T05:02:59.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:58 smithi005 ceph-mon[91801]: pgmap v1099: 105 pgs: 105 active+clean; 901 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 510 KiB/s rd, 32 MiB/s wr, 904 op/s 2024-04-26T05:02:59.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:58 smithi005 ceph-mon[91801]: osdmap e940: 8 total, 8 up, 8 in 2024-04-26T05:02:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:58 smithi111 ceph-mon[86360]: pgmap v1099: 105 pgs: 105 active+clean; 901 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 510 KiB/s rd, 32 MiB/s wr, 904 op/s 2024-04-26T05:02:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:58 smithi111 ceph-mon[86360]: osdmap e940: 8 total, 8 up, 8 in 2024-04-26T05:02:59.962 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:59.962 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:02:59.982 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:02:59.982 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:59.982 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:02:59.990 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:59.991 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.991 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.998 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:02:59.998 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:02:59.998 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.004 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:00.004 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.004 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.010 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:00.011 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.011 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.016 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:00.016 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.016 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.020 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:00.021 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.021 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.025 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:00.025 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.025 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:00.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:00.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:00.032 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:03:00.033 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:00.033 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:00.035 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:03:00.035 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:00.036 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:00.038 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:00.038 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:00.038 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:00.040 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:00.040 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:03:00.040 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:03:00.042 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~12582912,230686720~8388608,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,377487360~8388608,402653184~12582912] 2024-04-26T05:03:00.042 INFO:tasks.workunit.client.0.smithi005.stdout:from 14 to HEAD diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:03:00.042 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~8388608,16777216~12582912,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~4194304,289406976~4194304,297795584~8388608,314572800~12582912,339738624~12582912,377487360~8388608,402653184~8388608] 2024-04-26T05:03:00.043 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,230686720~8388608,251658240~12582912,276824064~8388608,289406976~4194304,297795584~8388608,314572800~12582912,339738624~16777216,377487360~8388608,402653184~8388608] 2024-04-26T05:03:00.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:02:59 smithi005 ceph-mon[91801]: osdmap e941: 8 total, 8 up, 8 in 2024-04-26T05:03:00.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:02:59 smithi005 ceph-mon[95809]: osdmap e941: 8 total, 8 up, 8 in 2024-04-26T05:03:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:02:59 smithi111 ceph-mon[86360]: osdmap e941: 8 total, 8 up, 8 in 2024-04-26T05:03:00.958 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:00.958 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:00.979 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:00.979 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:00.979 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:00.986 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:00.986 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.986 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.992 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:00.992 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.993 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.999 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:00.999 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:00.999 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.005 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:01.005 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.005 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.009 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:01.009 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.009 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.013 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:01.013 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.013 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.017 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:01.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:01.020 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:01.021 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:01.021 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:01.024 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:03:01.024 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:01.024 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:01.027 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:03:01.027 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:01.027 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:01.029 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:01.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:01.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:01.031 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:01.031 INFO:tasks.workunit.client.0.smithi005.stdout:from 13 to HEAD diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:03:01.031 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~25165824,41943040~12582912,67108864~4194304,75497472~16777216,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~8388608,230686720~8388608,251658240~12582912,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,377487360~8388608,402653184~12582912] 2024-04-26T05:03:01.032 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~12582912,230686720~8388608,251658240~12582912,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,377487360~8388608,402653184~12582912] 2024-04-26T05:03:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:00 smithi005 ceph-mon[91801]: pgmap v1102: 105 pgs: 105 active+clean; 901 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 510 KiB/s rd, 32 MiB/s wr, 904 op/s 2024-04-26T05:03:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:00 smithi005 ceph-mon[91801]: osdmap e942: 8 total, 8 up, 8 in 2024-04-26T05:03:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:00 smithi005 ceph-mon[95809]: pgmap v1102: 105 pgs: 105 active+clean; 901 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 510 KiB/s rd, 32 MiB/s wr, 904 op/s 2024-04-26T05:03:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:00 smithi005 ceph-mon[95809]: osdmap e942: 8 total, 8 up, 8 in 2024-04-26T05:03:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:00 smithi111 ceph-mon[86360]: pgmap v1102: 105 pgs: 105 active+clean; 901 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 510 KiB/s rd, 32 MiB/s wr, 904 op/s 2024-04-26T05:03:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:00 smithi111 ceph-mon[86360]: osdmap e942: 8 total, 8 up, 8 in 2024-04-26T05:03:01.976 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:01.976 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:01.995 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:01.995 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:01.995 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:02.001 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:02.001 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.001 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.006 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:02.006 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.006 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.012 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:02.012 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.012 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.017 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.017 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.017 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.021 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.021 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.021 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.025 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.025 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.025 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.028 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.029 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.029 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.031 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.031 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.031 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.034 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:03:02.034 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.034 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.036 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:03:02.036 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.036 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.038 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.038 INFO:tasks.workunit.client.0.smithi005.stdout:from 12 to HEAD diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.038 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,67108864~4194304,75497472~20971520,100663296~4194304,113246208~16777216,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.039 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~20971520,67108864~4194304,75497472~29360128,113246208~16777216,138412032~37748736,180355072~8388608,192937984~12582912,213909504~25165824,247463936~16777216,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:01 smithi005 ceph-mon[91801]: osdmap e943: 8 total, 8 up, 8 in 2024-04-26T05:03:02.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:01 smithi005 ceph-mon[95809]: osdmap e943: 8 total, 8 up, 8 in 2024-04-26T05:03:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:01 smithi111 ceph-mon[86360]: osdmap e943: 8 total, 8 up, 8 in 2024-04-26T05:03:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:02.792 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:02.792 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:02.811 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:02.812 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:02.812 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:02.817 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:02.817 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.817 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.823 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:02.823 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.823 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.827 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:02.827 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.827 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.831 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.831 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.831 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.835 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.835 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.835 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.839 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.839 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.839 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.842 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.842 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.842 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:02.844 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:02.845 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.845 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.847 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:03:02.847 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.847 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:02.849 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:03:02.849 INFO:tasks.workunit.client.0.smithi005.stdout:from 11 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.849 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~4194304,109051904~20971520,138412032~4194304,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~12582912,402653184~12582912] 2024-04-26T05:03:02.850 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~29360128,109051904~20971520,138412032~37748736,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~12582912,402653184~12582912] 2024-04-26T05:03:03.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:03:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:03:03] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2024-04-26T05:03:03.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:02 smithi005 ceph-mon[91801]: pgmap v1105: 105 pgs: 105 active+clean; 887 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 277 KiB/s rd, 1.5 KiB/s wr, 555 op/s 2024-04-26T05:03:03.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:02 smithi005 ceph-mon[91801]: osdmap e944: 8 total, 8 up, 8 in 2024-04-26T05:03:03.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:02 smithi005 ceph-mon[91801]: osdmap e945: 8 total, 8 up, 8 in 2024-04-26T05:03:03.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:02 smithi005 ceph-mon[95809]: pgmap v1105: 105 pgs: 105 active+clean; 887 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 277 KiB/s rd, 1.5 KiB/s wr, 555 op/s 2024-04-26T05:03:03.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:02 smithi005 ceph-mon[95809]: osdmap e944: 8 total, 8 up, 8 in 2024-04-26T05:03:03.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:02 smithi005 ceph-mon[95809]: osdmap e945: 8 total, 8 up, 8 in 2024-04-26T05:03:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:02 smithi111 ceph-mon[86360]: pgmap v1105: 105 pgs: 105 active+clean; 887 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 277 KiB/s rd, 1.5 KiB/s wr, 555 op/s 2024-04-26T05:03:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:02 smithi111 ceph-mon[86360]: osdmap e944: 8 total, 8 up, 8 in 2024-04-26T05:03:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:02 smithi111 ceph-mon[86360]: osdmap e945: 8 total, 8 up, 8 in 2024-04-26T05:03:03.793 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:03.794 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:03.813 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:03.813 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:03.813 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:03.818 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:03.818 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.818 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.824 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:03.824 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.824 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.829 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:03.829 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.829 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.833 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:03.833 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.833 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.837 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:03.837 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.837 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.840 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:03.841 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.841 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.843 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:03.843 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.844 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:03.846 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:03.846 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:03.846 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:03.848 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:03:03.848 INFO:tasks.workunit.client.0.smithi005.stdout:from 10 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:03.848 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~16777216,167772160~4194304,180355072~8388608,192937984~12582912,213909504~50331648,276824064~4194304,285212672~4194304,297795584~8388608,314572800~12582912,335544320~16777216,373293056~20971520,402653184~12582912] 2024-04-26T05:03:03.849 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~8388608,192937984~12582912,213909504~50331648,276824064~16777216,297795584~8388608,314572800~12582912,335544320~20971520,373293056~25165824,402653184~12582912] 2024-04-26T05:03:04.800 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:04.801 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:04.819 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:04.819 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:04.819 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:04.824 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:04.825 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.825 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.828 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:04.828 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.828 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.832 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:04.832 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.832 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.836 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:04.836 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.836 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.839 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:04.839 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.839 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.842 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:04.842 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.842 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.844 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:04.844 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.844 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:04.845 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:04.845 INFO:tasks.workunit.client.0.smithi005.stdout:from 9 to HEAD diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:04.846 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [4194304~29360128,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,402653184~12582912] 2024-04-26T05:03:04.846 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [4194304~33554432,41943040~29360128,75497472~100663296,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~25165824,402653184~12582912] 2024-04-26T05:03:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:04 smithi005 ceph-mon[95809]: pgmap v1108: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 509 KiB/s rd, 6.5 KiB/s wr, 935 op/s 2024-04-26T05:03:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:04 smithi005 ceph-mon[95809]: osdmap e946: 8 total, 8 up, 8 in 2024-04-26T05:03:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:04 smithi005 ceph-mon[91801]: pgmap v1108: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 509 KiB/s rd, 6.5 KiB/s wr, 935 op/s 2024-04-26T05:03:05.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:04 smithi005 ceph-mon[91801]: osdmap e946: 8 total, 8 up, 8 in 2024-04-26T05:03:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:04 smithi111 ceph-mon[86360]: pgmap v1108: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 509 KiB/s rd, 6.5 KiB/s wr, 935 op/s 2024-04-26T05:03:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:04 smithi111 ceph-mon[86360]: osdmap e946: 8 total, 8 up, 8 in 2024-04-26T05:03:05.798 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:05.798 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:05.817 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:05.818 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:05.818 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:05.827 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:05.827 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.827 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.831 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:05.831 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.831 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.835 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:05.835 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.835 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.837 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:05.837 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.837 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.840 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:05.840 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.840 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.842 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:05.842 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.842 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.844 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:05.844 INFO:tasks.workunit.client.0.smithi005.stdout:from 8 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.844 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~29360128,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:05.845 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:05 smithi005 ceph-mon[91801]: osdmap e947: 8 total, 8 up, 8 in 2024-04-26T05:03:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:05 smithi005 ceph-mon[95809]: osdmap e947: 8 total, 8 up, 8 in 2024-04-26T05:03:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:05 smithi111 ceph-mon[86360]: osdmap e947: 8 total, 8 up, 8 in 2024-04-26T05:03:06.813 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:06.814 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:06.830 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:06.830 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:06.830 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:06.834 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:06.834 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.834 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.837 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:06.837 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.837 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.840 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:06.840 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.840 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.843 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:06.843 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.843 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.845 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:06.845 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.845 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.848 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:06.848 INFO:tasks.workunit.client.0.smithi005.stdout:from 7 to HEAD diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.848 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~33554432,41943040~16777216,62914560~8388608,75497472~20971520,100663296~41943040,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:06.849 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:07.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:06 smithi005 ceph-mon[91801]: pgmap v1111: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 5.0 KiB/s wr, 379 op/s 2024-04-26T05:03:07.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:06 smithi005 ceph-mon[91801]: osdmap e948: 8 total, 8 up, 8 in 2024-04-26T05:03:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:06 smithi005 ceph-mon[95809]: pgmap v1111: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 5.0 KiB/s wr, 379 op/s 2024-04-26T05:03:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:06 smithi005 ceph-mon[95809]: osdmap e948: 8 total, 8 up, 8 in 2024-04-26T05:03:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:06 smithi111 ceph-mon[86360]: pgmap v1111: 105 pgs: 105 active+clean; 858 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 5.0 KiB/s wr, 379 op/s 2024-04-26T05:03:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:06 smithi111 ceph-mon[86360]: osdmap e948: 8 total, 8 up, 8 in 2024-04-26T05:03:07.796 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:07.796 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:07.817 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:07.817 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:07.817 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:07.820 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:07.821 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.821 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.824 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:07.824 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.824 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.826 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:07.826 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.826 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.828 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:07.828 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.828 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.829 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:07.830 INFO:tasks.workunit.client.0.smithi005.stdout:from 6 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.830 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,276824064~12582912,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:07.830 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~92274688,276824064~16777216,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:08.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:07 smithi005 ceph-mon[91801]: osdmap e949: 8 total, 8 up, 8 in 2024-04-26T05:03:08.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:07 smithi005 ceph-mon[91801]: osdmap e950: 8 total, 8 up, 8 in 2024-04-26T05:03:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:07 smithi005 ceph-mon[95809]: osdmap e949: 8 total, 8 up, 8 in 2024-04-26T05:03:08.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:07 smithi005 ceph-mon[95809]: osdmap e950: 8 total, 8 up, 8 in 2024-04-26T05:03:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:07 smithi111 ceph-mon[86360]: osdmap e949: 8 total, 8 up, 8 in 2024-04-26T05:03:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:07 smithi111 ceph-mon[86360]: osdmap e950: 8 total, 8 up, 8 in 2024-04-26T05:03:08.821 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:08.822 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:08.839 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:08.840 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:08.840 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:08.845 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:08.845 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.845 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.849 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:08.849 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.849 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.851 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:08.851 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.851 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.853 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:08.853 INFO:tasks.workunit.client.0.smithi005.stdout:from 5 to HEAD diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.853 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~37748736,41943040~16777216,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~16777216,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:08.854 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~37748736,41943040~134217728,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:09.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:08 smithi005 ceph-mon[91801]: pgmap v1114: 105 pgs: 105 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 6.2 KiB/s wr, 364 op/s 2024-04-26T05:03:09.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:08 smithi005 ceph-mon[95809]: pgmap v1114: 105 pgs: 105 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 6.2 KiB/s wr, 364 op/s 2024-04-26T05:03:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:08 smithi111 ceph-mon[86360]: pgmap v1114: 105 pgs: 105 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 6.2 KiB/s wr, 364 op/s 2024-04-26T05:03:09.833 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:09.833 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:09.849 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:09.849 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:09.849 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:09.851 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:09.853 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:09.853 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:09.854 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:09.854 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:09.854 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:09.856 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:09.856 INFO:tasks.workunit.client.0.smithi005.stdout:from 4 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:09.856 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,373293056~20971520,398458880~16777216] 2024-04-26T05:03:09.857 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~113246208,297795584~8388608,314572800~46137344,373293056~41943040] 2024-04-26T05:03:10.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:09 smithi005 ceph-mon[91801]: osdmap e951: 8 total, 8 up, 8 in 2024-04-26T05:03:10.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:09 smithi005 ceph-mon[95809]: osdmap e951: 8 total, 8 up, 8 in 2024-04-26T05:03:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:09 smithi111 ceph-mon[86360]: osdmap e951: 8 total, 8 up, 8 in 2024-04-26T05:03:10.830 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:10.830 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:10.843 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:10.843 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:10.843 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:10.848 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:10.848 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:10.848 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:10.851 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:10.851 INFO:tasks.workunit.client.0.smithi005.stdout:from 3 to HEAD diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:10.851 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~79691776,146800640~25165824,180355072~29360128,213909504~54525952,272629760~20971520,297795584~8388608,314572800~41943040,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:10.854 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:10 smithi005 ceph-mon[91801]: pgmap v1117: 105 pgs: 105 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 6.2 KiB/s wr, 364 op/s 2024-04-26T05:03:11.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:10 smithi005 ceph-mon[91801]: osdmap e952: 8 total, 8 up, 8 in 2024-04-26T05:03:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:10 smithi005 ceph-mon[95809]: pgmap v1117: 105 pgs: 105 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 6.2 KiB/s wr, 364 op/s 2024-04-26T05:03:11.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:10 smithi005 ceph-mon[95809]: osdmap e952: 8 total, 8 up, 8 in 2024-04-26T05:03:11.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:10 smithi111 ceph-mon[86360]: pgmap v1117: 105 pgs: 105 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 6.2 KiB/s wr, 364 op/s 2024-04-26T05:03:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:10 smithi111 ceph-mon[86360]: osdmap e952: 8 total, 8 up, 8 in 2024-04-26T05:03:11.840 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:11.840 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:11.852 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:11.852 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:11.852 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:11.854 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:11.855 INFO:tasks.workunit.client.0.smithi005.stdout:from 2 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:11.855 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~29360128,314572800~46137344,364904448~4194304,373293056~20971520,398458880~16777216] 2024-04-26T05:03:11.857 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:12.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:11 smithi005 ceph-mon[91801]: osdmap e953: 8 total, 8 up, 8 in 2024-04-26T05:03:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:11 smithi005 ceph-mon[95809]: osdmap e953: 8 total, 8 up, 8 in 2024-04-26T05:03:12.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:11 smithi111 ceph-mon[86360]: osdmap e953: 8 total, 8 up, 8 in 2024-04-26T05:03:12.790 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:12.790 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:12.802 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:12.802 INFO:tasks.workunit.client.0.smithi005.stdout:from 1 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:12.802 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~4194304,373293056~41943040] 2024-04-26T05:03:12.806 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~54525952,373293056~41943040] 2024-04-26T05:03:13.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:12 smithi005 ceph-mon[95809]: pgmap v1120: 105 pgs: 105 active+clean; 684 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 767 B/s wr, 57 op/s 2024-04-26T05:03:13.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:12 smithi005 ceph-mon[95809]: osdmap e954: 8 total, 8 up, 8 in 2024-04-26T05:03:13.131 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:12 smithi005 ceph-mon[95809]: osdmap e955: 8 total, 8 up, 8 in 2024-04-26T05:03:13.131 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:12 smithi005 ceph-mon[91801]: pgmap v1120: 105 pgs: 105 active+clean; 684 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 767 B/s wr, 57 op/s 2024-04-26T05:03:13.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:12 smithi005 ceph-mon[91801]: osdmap e954: 8 total, 8 up, 8 in 2024-04-26T05:03:13.132 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:12 smithi005 ceph-mon[91801]: osdmap e955: 8 total, 8 up, 8 in 2024-04-26T05:03:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:12 smithi111 ceph-mon[86360]: pgmap v1120: 105 pgs: 105 active+clean; 684 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 767 B/s wr, 57 op/s 2024-04-26T05:03:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:12 smithi111 ceph-mon[86360]: osdmap e954: 8 total, 8 up, 8 in 2024-04-26T05:03:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:12 smithi111 ceph-mon[86360]: osdmap e955: 8 total, 8 up, 8 in 2024-04-26T05:03:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:03:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:03:13] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2024-04-26T05:03:13.795 INFO:tasks.workunit.client.0.smithi005.stdout:from 0 to HEAD diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:13.795 INFO:tasks.workunit.client.0.smithi005.stdout: limited diff [0~58720256,62914560~8388608,75497472~67108864,146800640~25165824,180355072~29360128,213909504~54525952,272629760~33554432,314572800~46137344,364904448~50331648] 2024-04-26T05:03:13.805 INFO:tasks.workunit.client.0.smithi005.stdout: actual was [0~176160768,180355072~125829120,314572800~100663296] 2024-04-26T05:03:14.812 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (38958 ms) 2024-04-26T05:03:14.812 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-04-26T05:03:14.812 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:15.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:14 smithi005 ceph-mon[95809]: pgmap v1123: 105 pgs: 105 active+clean; 574 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 108 KiB/s rd, 5.7 KiB/s wr, 174 op/s 2024-04-26T05:03:15.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:14 smithi005 ceph-mon[95809]: osdmap e956: 8 total, 8 up, 8 in 2024-04-26T05:03:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:14 smithi005 ceph-mon[91801]: pgmap v1123: 105 pgs: 105 active+clean; 574 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 108 KiB/s rd, 5.7 KiB/s wr, 174 op/s 2024-04-26T05:03:15.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:14 smithi005 ceph-mon[91801]: osdmap e956: 8 total, 8 up, 8 in 2024-04-26T05:03:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:14 smithi111 ceph-mon[86360]: pgmap v1123: 105 pgs: 105 active+clean; 574 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 108 KiB/s rd, 5.7 KiB/s wr, 174 op/s 2024-04-26T05:03:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:14 smithi111 ceph-mon[86360]: osdmap e956: 8 total, 8 up, 8 in 2024-04-26T05:03:15.908 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1096 ms) 2024-04-26T05:03:15.908 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-04-26T05:03:15.908 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:16.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:15 smithi005 ceph-mon[91801]: osdmap e957: 8 total, 8 up, 8 in 2024-04-26T05:03:16.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:15 smithi005 ceph-mon[95809]: osdmap e957: 8 total, 8 up, 8 in 2024-04-26T05:03:16.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:15 smithi111 ceph-mon[86360]: osdmap e957: 8 total, 8 up, 8 in 2024-04-26T05:03:16.926 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [0~20971520] to clone 2024-04-26T05:03:16.926 INFO:tasks.workunit.client.0.smithi005.stdout: diff was [0~20971520] 2024-04-26T05:03:16.930 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1022 ms) 2024-04-26T05:03:16.930 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-04-26T05:03:16.930 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:16.949 INFO:tasks.workunit.client.0.smithi005.stdout: wrote [885618~49146,3344829~60157,3676076~55592,4998767~63683,6733814~12042,7215241~58445,7866730~34266,11849513~78036,15085394~20957,15954174~53247] 2024-04-26T05:03:16.958 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (28 ms) 2024-04-26T05:03:16.958 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-04-26T05:03:16.958 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:17.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:16 smithi005 ceph-mon[95809]: pgmap v1126: 105 pgs: 105 active+clean; 574 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 117 op/s 2024-04-26T05:03:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:16 smithi005 ceph-mon[95809]: osdmap e958: 8 total, 8 up, 8 in 2024-04-26T05:03:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:16 smithi005 ceph-mon[91801]: pgmap v1126: 105 pgs: 105 active+clean; 574 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 117 op/s 2024-04-26T05:03:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:16 smithi005 ceph-mon[91801]: osdmap e958: 8 total, 8 up, 8 in 2024-04-26T05:03:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:16 smithi111 ceph-mon[86360]: pgmap v1126: 105 pgs: 105 active+clean; 574 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 117 op/s 2024-04-26T05:03:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:16 smithi111 ceph-mon[86360]: osdmap e958: 8 total, 8 up, 8 in 2024-04-26T05:03:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:18.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:17 smithi005 ceph-mon[91801]: osdmap e959: 8 total, 8 up, 8 in 2024-04-26T05:03:18.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:17 smithi005 ceph-mon[91801]: osdmap e960: 8 total, 8 up, 8 in 2024-04-26T05:03:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:17 smithi005 ceph-mon[95809]: osdmap e959: 8 total, 8 up, 8 in 2024-04-26T05:03:18.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:17 smithi005 ceph-mon[95809]: osdmap e960: 8 total, 8 up, 8 in 2024-04-26T05:03:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:17 smithi111 ceph-mon[86360]: osdmap e959: 8 total, 8 up, 8 in 2024-04-26T05:03:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:17 smithi111 ceph-mon[86360]: osdmap e960: 8 total, 8 up, 8 in 2024-04-26T05:03:18.911 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1953 ms) 2024-04-26T05:03:18.911 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-04-26T05:03:18.911 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:18.973 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (62 ms) 2024-04-26T05:03:18.974 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-04-26T05:03:18.974 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:19.068 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (95 ms) 2024-04-26T05:03:19.068 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTryAcquireLock 2024-04-26T05:03:19.069 INFO:tasks.workunit.client.0.smithi005.stdout:using new format! 2024-04-26T05:03:19.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:18 smithi005 ceph-mon[91801]: pgmap v1129: 105 pgs: 105 active+clean; 494 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 265 op/s 2024-04-26T05:03:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:18 smithi005 ceph-mon[95809]: pgmap v1129: 105 pgs: 105 active+clean; 494 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 265 op/s 2024-04-26T05:03:19.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:18 smithi111 ceph-mon[86360]: pgmap v1129: 105 pgs: 105 active+clean; 494 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 265 op/s 2024-04-26T05:03:20.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:19 smithi005 ceph-mon[91801]: osdmap e961: 8 total, 8 up, 8 in 2024-04-26T05:03:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:19 smithi005 ceph-mon[95809]: osdmap e961: 8 total, 8 up, 8 in 2024-04-26T05:03:20.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:19 smithi111 ceph-mon[86360]: osdmap e961: 8 total, 8 up, 8 in 2024-04-26T05:03:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:20 smithi111 ceph-mon[86360]: pgmap v1132: 105 pgs: 105 active+clean; 494 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 265 op/s 2024-04-26T05:03:21.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:20 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:20 smithi005 ceph-mon[91801]: pgmap v1132: 105 pgs: 105 active+clean; 494 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 265 op/s 2024-04-26T05:03:21.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:20 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:20 smithi005 ceph-mon[95809]: pgmap v1132: 105 pgs: 105 active+clean; 494 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 265 op/s 2024-04-26T05:03:21.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:20 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:22 smithi005 ceph-mon[95809]: pgmap v1133: 105 pgs: 105 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.3 MiB/s wr, 217 op/s 2024-04-26T05:03:22.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:22 smithi005 ceph-mon[95809]: osdmap e962: 8 total, 8 up, 8 in 2024-04-26T05:03:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:22 smithi005 ceph-mon[91801]: pgmap v1133: 105 pgs: 105 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.3 MiB/s wr, 217 op/s 2024-04-26T05:03:22.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:22 smithi005 ceph-mon[91801]: osdmap e962: 8 total, 8 up, 8 in 2024-04-26T05:03:23.232 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:22 smithi111 ceph-mon[86360]: pgmap v1133: 105 pgs: 105 active+clean; 530 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.3 MiB/s wr, 217 op/s 2024-04-26T05:03:23.233 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:22 smithi111 ceph-mon[86360]: osdmap e962: 8 total, 8 up, 8 in 2024-04-26T05:03:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:03:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:03:23] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2024-04-26T05:03:24.109 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateTryAcquireLock (5041 ms) 2024-04-26T05:03:24.109 INFO:tasks.workunit.client.0.smithi005.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-04-26T05:03:24.109 INFO:tasks.workunit.client.0.smithi005.stdout:SKIPPING 2024-04-26T05:03:24.109 INFO:tasks.workunit.client.0.smithi005.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] 15 tests from DiffIterateTest/1 (68229 ms total) 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout:[----------] Global test environment tear-down 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout:[==========] 415 tests from 16 test suites ran. (1000624 ms total) 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout:[ PASSED ] 415 tests. 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout: YOU HAVE 2 DISABLED TESTS 2024-04-26T05:03:24.940 INFO:tasks.workunit.client.0.smithi005.stdout: 2024-04-26T05:03:24.945 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-26T05:03:24.946 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-26T05:03:25.013 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-04-26T05:03:25.013 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: pgmap v1135: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 5.8 MiB/s wr, 125 op/s 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:25.074 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: pgmap v1135: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 5.8 MiB/s wr, 125 op/s 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:25.075 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:24 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: pgmap v1135: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 5.8 MiB/s wr, 125 op/s 2024-04-26T05:03:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:25.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:25.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:24 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:03:25.271 DEBUG:teuthology.parallel:result is None 2024-04-26T05:03:25.271 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T05:03:25.335 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T05:03:25.335 DEBUG:teuthology.orchestra.run.smithi005:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T05:03:25.397 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T05:03:25.397 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-26T05:03:25.397 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-04-26T05:03:25.397 DEBUG:teuthology.parallel:result is None 2024-04-26T05:03:25.398 INFO:teuthology.run_tasks:Running task print... 2024-04-26T05:03:25.406 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-26T05:03:25.407 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T05:03:25.417 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T05:03:25.417 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T05:03:25.418 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T05:03:25.703 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:26.151 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:25 smithi005 ceph-mon[91801]: pgmap v1136: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.2 MiB/s wr, 111 op/s 2024-04-26T05:03:26.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:25 smithi005 ceph-mon[91801]: pgmap v1137: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 6.1 MiB/s wr, 130 op/s 2024-04-26T05:03:26.152 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:25 smithi005 ceph-mon[91801]: osdmap e963: 8 total, 8 up, 8 in 2024-04-26T05:03:26.152 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:25 smithi005 ceph-mon[95809]: pgmap v1136: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.2 MiB/s wr, 111 op/s 2024-04-26T05:03:26.152 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:25 smithi005 ceph-mon[95809]: pgmap v1137: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 6.1 MiB/s wr, 130 op/s 2024-04-26T05:03:26.152 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:25 smithi005 ceph-mon[95809]: osdmap e963: 8 total, 8 up, 8 in 2024-04-26T05:03:26.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:25 smithi111 ceph-mon[86360]: pgmap v1136: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.2 MiB/s wr, 111 op/s 2024-04-26T05:03:26.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:25 smithi111 ceph-mon[86360]: pgmap v1137: 105 pgs: 105 active+clean; 557 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s rd, 6.1 MiB/s wr, 130 op/s 2024-04-26T05:03:26.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:25 smithi111 ceph-mon[86360]: osdmap e963: 8 total, 8 up, 8 in 2024-04-26T05:03:27.694 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:03:28.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:27 smithi005 ceph-mon[91801]: pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:27 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/399811394' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:03:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:27 smithi005 ceph-mon[95809]: pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:27 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/399811394' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:03:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:27 smithi111 ceph-mon[86360]: pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:27 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/399811394' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:03:28.357 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":38,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6800","nonce":3568076589},{"type":"v1","addr":"172.21.15.5:6801","nonce":3568076589}]},"active_addr":"172.21.15.5:6801/3568076589","active_change":"2024-04-26T04:50:01.492197+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25228,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.5:8443/","prometheus":"http://172.21.15.5:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":262,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":1218551015}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":590981055}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":3919291695}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.5:0","nonce":238566116}]}]} 2024-04-26T05:03:28.361 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T05:03:28.361 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T05:03:28.361 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:03:28.631 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:30.056 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:29 smithi005 ceph-mon[95809]: pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.056 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:29 smithi005 ceph-mon[91801]: pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:29 smithi111 ceph-mon[86360]: pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.515 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:03:30.515 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":963,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T05:03:24.932698+0000","last_up_change":"2024-04-26T04:56:47.253320+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":41,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:44:07.074856+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:45:20.482057+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:45:21.051786+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":508,"up_from":519,"up_thru":899,"down_at":509,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6803","nonce":752873951}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6805","nonce":752873951}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6809","nonce":752873951}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6807","nonce":752873951}]},"public_addr":"172.21.15.5:6803/752873951","cluster_addr":"172.21.15.5:6805/752873951","heartbeat_back_addr":"172.21.15.5:6809/752873951","heartbeat_front_addr":"172.21.15.5:6807/752873951","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":542,"up_from":553,"up_thru":899,"down_at":543,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6811","nonce":3934897551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6813","nonce":3934897551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6817","nonce":3934897551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6815","nonce":3934897551}]},"public_addr":"172.21.15.5:6811/3934897551","cluster_addr":"172.21.15.5:6813/3934897551","heartbeat_back_addr":"172.21.15.5:6817/3934897551","heartbeat_front_addr":"172.21.15.5:6815/3934897551","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":607,"up_from":618,"up_thru":899,"down_at":608,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6819","nonce":1259178969}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6821","nonce":1259178969}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6825","nonce":1259178969}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6823","nonce":1259178969}]},"public_addr":"172.21.15.5:6819/1259178969","cluster_addr":"172.21.15.5:6821/1259178969","heartbeat_back_addr":"172.21.15.5:6825/1259178969","heartbeat_front_addr":"172.21.15.5:6823/1259178969","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":630,"up_from":636,"up_thru":899,"down_at":630,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6827","nonce":2840627157}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6829","nonce":2840627157}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6833","nonce":2840627157}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6831","nonce":2840627157}]},"public_addr":"172.21.15.5:6827/2840627157","cluster_addr":"172.21.15.5:6829/2840627157","heartbeat_back_addr":"172.21.15.5:6833/2840627157","heartbeat_front_addr":"172.21.15.5:6831/2840627157","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":508},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":542},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":606},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":630},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/1045584742":"2024-04-26T05:59:01.507621+0000","172.21.15.5:0/209732525":"2024-04-26T05:54:47.728346+0000","172.21.15.5:0/2708411967":"2024-04-26T05:54:46.723763+0000","172.21.15.5:0/1961539283":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/188422345":"2024-04-27T04:48:07.165349+0000","172.21.15.5:6800/2567873487":"2024-04-27T04:50:01.491964+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4211393178":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.111:0/3183530232":"2024-04-27T04:49:14.729301+0000","172.21.15.111:0/2286493110":"2024-04-27T04:49:14.729301+0000","172.21.15.5:0/3606668421":"2024-04-27T04:50:01.491964+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.111:0/1814529578":"2024-04-27T04:49:14.729301+0000","172.21.15.5:0/1663053230":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3920149860":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/2166298037":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4008904229":"2024-04-27T04:48:07.165349+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.111:0/837020546":"2024-04-27T04:49:14.729301+0000","172.21.15.5:6801/1913329511":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/3749470361":"2024-04-27T04:50:01.491964+0000","172.21.15.111:6834/2263081650":"2024-04-27T04:49:14.729301+0000","172.21.15.111:6835/2263081650":"2024-04-27T04:49:14.729301+0000","172.21.15.5:6800/1913329511":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/1382772406":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/105735948":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/2528052628":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/3928321375":"2024-04-27T04:44:05.460958+0000","172.21.15.111:0/1060453066":"2024-04-27T04:49:14.729301+0000","172.21.15.5:6801/2567873487":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.111:0/1315556088":"2024-04-27T04:49:14.729301+0000","172.21.15.5:0/2529091971":"2024-04-27T04:50:01.491964+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":4,"length":5}]},{"pool":16,"snaps":[{"begin":34,"length":4},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:03:31.149 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T05:03:31.149 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:03:31.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:30 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/168055352' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:03:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:30 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/168055352' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:03:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:30 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/168055352' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:03:31.418 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:32.210 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:31 smithi005 ceph-mon[91801]: pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.211 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:32.211 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:31 smithi005 ceph-mon[95809]: pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.211 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:31 smithi111 ceph-mon[86360]: pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:33.245 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:03:33.245 INFO:teuthology.orchestra.run.smithi005.stdout:{"epoch":963,"fsid":"b5a9bf3e-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:38:21.942634+0000","modified":"2024-04-26T05:03:24.932698+0000","last_up_change":"2024-04-26T04:56:47.253320+0000","last_in_change":"2024-04-26T04:43:15.262985+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":41,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:41:33.442542+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:44:07.074856+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:45:20.482057+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:45:21.051786+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"daa9ec57-3f2d-4901-9460-dae474808678","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":508,"up_from":519,"up_thru":899,"down_at":509,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6802","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6803","nonce":752873951}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6804","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6805","nonce":752873951}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6808","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6809","nonce":752873951}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6806","nonce":752873951},{"type":"v1","addr":"172.21.15.5:6807","nonce":752873951}]},"public_addr":"172.21.15.5:6803/752873951","cluster_addr":"172.21.15.5:6805/752873951","heartbeat_back_addr":"172.21.15.5:6809/752873951","heartbeat_front_addr":"172.21.15.5:6807/752873951","state":["exists","up"]},{"osd":1,"uuid":"ef176dd7-ec4e-4f2f-91a9-f5cbe0eb07aa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":542,"up_from":553,"up_thru":899,"down_at":543,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6810","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6811","nonce":3934897551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6812","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6813","nonce":3934897551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6816","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6817","nonce":3934897551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6814","nonce":3934897551},{"type":"v1","addr":"172.21.15.5:6815","nonce":3934897551}]},"public_addr":"172.21.15.5:6811/3934897551","cluster_addr":"172.21.15.5:6813/3934897551","heartbeat_back_addr":"172.21.15.5:6817/3934897551","heartbeat_front_addr":"172.21.15.5:6815/3934897551","state":["exists","up"]},{"osd":2,"uuid":"d89110dd-b5fa-45e6-a733-76e6b536be58","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":607,"up_from":618,"up_thru":899,"down_at":608,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6818","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6819","nonce":1259178969}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6820","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6821","nonce":1259178969}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6824","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6825","nonce":1259178969}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6822","nonce":1259178969},{"type":"v1","addr":"172.21.15.5:6823","nonce":1259178969}]},"public_addr":"172.21.15.5:6819/1259178969","cluster_addr":"172.21.15.5:6821/1259178969","heartbeat_back_addr":"172.21.15.5:6825/1259178969","heartbeat_front_addr":"172.21.15.5:6823/1259178969","state":["exists","up"]},{"osd":3,"uuid":"06dcdf4a-cd71-407e-b50d-5a6ebc5535ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":630,"up_from":636,"up_thru":899,"down_at":630,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6826","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6827","nonce":2840627157}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6828","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6829","nonce":2840627157}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6832","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6833","nonce":2840627157}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.5:6830","nonce":2840627157},{"type":"v1","addr":"172.21.15.5:6831","nonce":2840627157}]},"public_addr":"172.21.15.5:6827/2840627157","cluster_addr":"172.21.15.5:6829/2840627157","heartbeat_back_addr":"172.21.15.5:6833/2840627157","heartbeat_front_addr":"172.21.15.5:6831/2840627157","state":["exists","up"]},{"osd":4,"uuid":"3fab3e60-abe9-4e0f-812c-4333ab56419c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6800","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6801","nonce":2939693462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6802","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6803","nonce":2939693462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6806","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6807","nonce":2939693462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6804","nonce":2939693462},{"type":"v1","addr":"172.21.15.111:6805","nonce":2939693462}]},"public_addr":"172.21.15.111:6801/2939693462","cluster_addr":"172.21.15.111:6803/2939693462","heartbeat_back_addr":"172.21.15.111:6807/2939693462","heartbeat_front_addr":"172.21.15.111:6805/2939693462","state":["exists","up"]},{"osd":5,"uuid":"888dcd16-78cf-4776-a4e3-4375e0c10e53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6808","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6809","nonce":251519988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6810","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6811","nonce":251519988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6814","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6815","nonce":251519988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6812","nonce":251519988},{"type":"v1","addr":"172.21.15.111:6813","nonce":251519988}]},"public_addr":"172.21.15.111:6809/251519988","cluster_addr":"172.21.15.111:6811/251519988","heartbeat_back_addr":"172.21.15.111:6815/251519988","heartbeat_front_addr":"172.21.15.111:6813/251519988","state":["exists","up"]},{"osd":6,"uuid":"0cdf3d72-f096-41f7-bbdb-c97483f0756d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6816","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6817","nonce":1634521235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6818","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6819","nonce":1634521235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6822","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6823","nonce":1634521235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6820","nonce":1634521235},{"type":"v1","addr":"172.21.15.111:6821","nonce":1634521235}]},"public_addr":"172.21.15.111:6817/1634521235","cluster_addr":"172.21.15.111:6819/1634521235","heartbeat_back_addr":"172.21.15.111:6823/1634521235","heartbeat_front_addr":"172.21.15.111:6821/1634521235","state":["exists","up"]},{"osd":7,"uuid":"dcc19600-f6e1-4c85-9f2a-8142ba69db4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6824","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6825","nonce":3024259509}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6826","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6827","nonce":3024259509}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6830","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6831","nonce":3024259509}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.111:6828","nonce":3024259509},{"type":"v1","addr":"172.21.15.111:6829","nonce":3024259509}]},"public_addr":"172.21.15.111:6825/3024259509","cluster_addr":"172.21.15.111:6827/3024259509","heartbeat_back_addr":"172.21.15.111:6831/3024259509","heartbeat_front_addr":"172.21.15.111:6829/3024259509","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:40:40.573253+0000","dead_epoch":508},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:04.825050+0000","dead_epoch":542},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:29.451815+0000","dead_epoch":606},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:41:54.992452+0000","dead_epoch":630},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:17.109835+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:42:41.678551+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:05.853396+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:43:30.532198+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.5:0/1045584742":"2024-04-26T05:59:01.507621+0000","172.21.15.5:0/209732525":"2024-04-26T05:54:47.728346+0000","172.21.15.5:0/2708411967":"2024-04-26T05:54:46.723763+0000","172.21.15.5:0/1961539283":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/3149744552":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/188422345":"2024-04-27T04:48:07.165349+0000","172.21.15.5:6800/2567873487":"2024-04-27T04:50:01.491964+0000","172.21.15.5:6801/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4211393178":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/2027006058":"2024-04-27T04:39:20.937465+0000","172.21.15.111:0/3183530232":"2024-04-27T04:49:14.729301+0000","172.21.15.111:0/2286493110":"2024-04-27T04:49:14.729301+0000","172.21.15.5:0/3606668421":"2024-04-27T04:50:01.491964+0000","172.21.15.5:6800/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/863457686":"2024-04-27T04:39:20.937465+0000","172.21.15.111:0/1814529578":"2024-04-27T04:49:14.729301+0000","172.21.15.5:0/1663053230":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/4192313389":"2024-04-27T04:39:20.937465+0000","172.21.15.5:0/3920149860":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/2166298037":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/3610558256":"2024-04-27T04:38:56.934026+0000","172.21.15.5:0/4008904229":"2024-04-27T04:48:07.165349+0000","172.21.15.5:6800/159159969":"2024-04-27T04:38:56.934026+0000","172.21.15.111:0/837020546":"2024-04-27T04:49:14.729301+0000","172.21.15.5:6801/1913329511":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/3749470361":"2024-04-27T04:50:01.491964+0000","172.21.15.111:6834/2263081650":"2024-04-27T04:49:14.729301+0000","172.21.15.111:6835/2263081650":"2024-04-27T04:49:14.729301+0000","172.21.15.5:6800/1913329511":"2024-04-27T04:48:07.165349+0000","172.21.15.5:0/1382772406":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/105735948":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/2528052628":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/3928321375":"2024-04-27T04:44:05.460958+0000","172.21.15.111:0/1060453066":"2024-04-27T04:49:14.729301+0000","172.21.15.5:6801/2567873487":"2024-04-27T04:50:01.491964+0000","172.21.15.5:0/1267826893":"2024-04-27T04:38:56.934026+0000","172.21.15.5:6800/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.5:0/3461662409":"2024-04-27T04:39:20.937465+0000","172.21.15.5:6801/404244191":"2024-04-27T04:44:05.460958+0000","172.21.15.111:0/1315556088":"2024-04-27T04:49:14.729301+0000","172.21.15.5:0/2529091971":"2024-04-27T04:50:01.491964+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":4,"length":5}]},{"pool":16,"snaps":[{"begin":34,"length":4},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:03:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:03:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:03:33] "GET /metrics HTTP/1.1" 200 34240 "" "Prometheus/2.43.0" 2024-04-26T05:03:33.891 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T05:03:33.892 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T05:03:33.892 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T05:03:33.892 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T05:03:33.893 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T05:03:33.893 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T05:03:33.894 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T05:03:33.894 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T05:03:34.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:33 smithi005 ceph-mon[91801]: pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:34.231 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:33 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4270058257' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:33 smithi005 ceph-mon[95809]: pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:34.231 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:33 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4270058257' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:33 smithi111 ceph-mon[86360]: pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:34.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:33 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4270058257' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.418 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.418 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.419 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.419 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.419 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.475 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.487 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:36.207 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:35 smithi005 ceph-mon[91801]: pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:36.207 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:35 smithi005 ceph-mon[95809]: pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:35 smithi111 ceph-mon[86360]: pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:37 smithi005 ceph-mon[91801]: pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:37 smithi005 ceph-mon[95809]: pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:38.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:37 smithi111 ceph-mon[86360]: pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:40.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:39 smithi005 ceph-mon[91801]: pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:40.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:39 smithi005 ceph-mon[95809]: pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:40.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:39 smithi111 ceph-mon[86360]: pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:41.444 INFO:teuthology.orchestra.run.smithi005.stdout:2375116914792 2024-04-26T05:03:41.445 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T05:03:41.589 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:42.225 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:41 smithi005 ceph-mon[91801]: pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:42.225 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:41 smithi005 ceph-mon[95809]: pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:42.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:41 smithi111 ceph-mon[86360]: pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:43.345 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:03:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:03:43] "GET /metrics HTTP/1.1" 200 34240 "" "Prometheus/2.43.0" 2024-04-26T05:03:44.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:43 smithi005 ceph-mon[91801]: pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:44.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:43 smithi005 ceph-mon[95809]: pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:44.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:43 smithi111 ceph-mon[86360]: pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:44.559 INFO:teuthology.orchestra.run.smithi005.stdout:2731599200341 2024-04-26T05:03:44.560 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T05:03:46.143 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:45 smithi005 ceph-mon[91801]: pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:46.144 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:45 smithi005 ceph-mon[95809]: pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:46.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:45 smithi111 ceph-mon[86360]: pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:46.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:46.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:47.361 INFO:teuthology.orchestra.run.smithi005.stdout:236223201526 2024-04-26T05:03:47.361 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T05:03:47.776 INFO:teuthology.orchestra.run.smithi005.stdout:201863463163 2024-04-26T05:03:47.776 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T05:03:47.931 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:48.053 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:47 smithi005 ceph-mon[91801]: pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:48.053 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:47 smithi005 ceph-mon[95809]: pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:47 smithi111 ceph-mon[86360]: pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:50.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:49 smithi005 ceph-mon[91801]: pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:50.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:49 smithi005 ceph-mon[95809]: pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:50.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:49 smithi111 ceph-mon[86360]: pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:51.208 INFO:teuthology.orchestra.run.smithi005.stdout:2654289789020 2024-04-26T05:03:51.209 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T05:03:51.250 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:52.069 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:51 smithi005 ceph-mon[91801]: pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:52.069 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:51 smithi005 ceph-mon[95809]: pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:52.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:51 smithi111 ceph-mon[86360]: pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:53.196 INFO:teuthology.orchestra.run.smithi005.stdout:133143986436 2024-04-26T05:03:53.197 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T05:03:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:03:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:03:53] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:03:53.852 INFO:teuthology.orchestra.run.smithi005.stdout:2229088026737 2024-04-26T05:03:53.852 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T05:03:54.085 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:53 smithi005 ceph-mon[95809]: pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:54.085 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:53 smithi005 ceph-mon[91801]: pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:53 smithi111 ceph-mon[86360]: pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:54.574 INFO:teuthology.orchestra.run.smithi005.stdout:2731599200343 2024-04-26T05:03:54.685 INFO:teuthology.orchestra.run.smithi005.stdout:167503724801 2024-04-26T05:03:54.685 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T05:03:55.081 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:55.082 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:55.130 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:55.214 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3875977824' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:03:55.214 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3875977824' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:03:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3875977824' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:03:55.303 INFO:teuthology.orchestra.run.smithi005.stdout:2375116914795 2024-04-26T05:03:56.211 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:55 smithi111 ceph-mon[86360]: pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:56.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:55 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4186135935' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:03:56.298 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:55 smithi005 ceph-mon[91801]: pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:56.298 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:55 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4186135935' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:03:56.298 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:55 smithi005 ceph-mon[95809]: pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:56.298 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:55 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4186135935' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:03:56.624 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2731599200341 got 2731599200343 for osd.3 2024-04-26T05:03:56.624 DEBUG:teuthology.parallel:result is None 2024-04-26T05:03:58.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:57 smithi111 ceph-mon[86360]: pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:58.324 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:57 smithi005 ceph-mon[91801]: pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:58.324 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:57 smithi005 ceph-mon[95809]: pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:03:58.986 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2375116914792 got 2375116914795 for osd.1 2024-04-26T05:03:58.986 DEBUG:teuthology.parallel:result is None 2024-04-26T05:03:59.421 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:00.263 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:03:59 smithi005 ceph-mon[95809]: pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:00.263 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:03:59 smithi005 ceph-mon[91801]: pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:03:59 smithi111 ceph-mon[86360]: pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:01.796 INFO:teuthology.orchestra.run.smithi005.stdout:236223201529 2024-04-26T05:04:01.800 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:01.924 INFO:teuthology.orchestra.run.smithi005.stdout:201863463166 2024-04-26T05:04:02.212 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[91801]: pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1846358512' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3438233302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[95809]: pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1846358512' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:04:02.213 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:01 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3438233302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:04:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:01 smithi111 ceph-mon[86360]: pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:01 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1846358512' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:04:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:01 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3438233302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:04:03.341 INFO:teuthology.orchestra.run.smithi005.stdout:2654289789024 2024-04-26T05:04:03.354 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:04:03] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:04:03.549 INFO:teuthology.orchestra.run.smithi005.stdout:2229088026740 2024-04-26T05:04:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:03 smithi111 ceph-mon[86360]: pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:03 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1064054356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:04:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:03 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4254810103' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:04:04.336 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-mon[91801]: pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:04.337 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1064054356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:04:04.337 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4254810103' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:04:04.337 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-mon[95809]: pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:04.337 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1064054356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:04:04.337 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:03 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4254810103' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:04:04.367 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463163 got 201863463166 for osd.6 2024-04-26T05:04:04.368 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:04.838 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201526 got 236223201529 for osd.7 2024-04-26T05:04:04.838 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:05.173 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2229088026737 got 2229088026740 for osd.0 2024-04-26T05:04:05.173 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:05.567 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2654289789020 got 2654289789024 for osd.2 2024-04-26T05:04:05.567 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:06.058 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:05 smithi005 ceph-mon[91801]: pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:06.059 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:05 smithi005 ceph-mon[95809]: pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:05 smithi111 ceph-mon[86360]: pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:07.561 INFO:teuthology.orchestra.run.smithi005.stdout:133143986439 2024-04-26T05:04:07.989 INFO:teuthology.orchestra.run.smithi005.stdout:167503724804 2024-04-26T05:04:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:07 smithi111 ceph-mon[86360]: pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:07 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/39464701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:04:08.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:07 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3862688272' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:04:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:07 smithi005 ceph-mon[91801]: pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:07 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/39464701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:04:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:07 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3862688272' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:04:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:07 smithi005 ceph-mon[95809]: pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:07 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/39464701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:04:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:07 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3862688272' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:04:08.525 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986436 got 133143986439 for osd.4 2024-04-26T05:04:08.525 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:08.958 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724801 got 167503724804 for osd.5 2024-04-26T05:04:08.958 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:08.958 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T05:04:08.958 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:04:09.231 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:10.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:10 smithi111 ceph-mon[86360]: pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:10.348 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:10 smithi005 ceph-mon[91801]: pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:10.348 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:10 smithi005 ceph-mon[95809]: pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:11.024 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:11.025 INFO:teuthology.orchestra.run.smithi005.stderr:dumped all 2024-04-26T05:04:11.719 INFO:teuthology.orchestra.run.smithi005.stdout:{"pg_ready":true,"pg_map":{"version":1161,"stamp":"2024-04-26T05:04:10.566754+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1547220,"kb_used_data":125172,"kb_used_omap":264,"kb_used_meta":1421687,"kb_avail":748413996,"statfs":{"total":767960285184,"available":766375931904,"internally_reserved":0,"allocated":128176128,"data_stored":79952128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":271020,"internal_metadata":1455807828},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003427"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150394+0000","last_change":"2024-04-26T04:55:08.143144+0000","last_active":"2024-04-26T05:03:23.150394+0000","last_peered":"2024-04-26T05:03:23.150394+0000","last_clean":"2024-04-26T05:03:23.150394+0000","last_became_active":"2024-04-26T04:55:08.142745+0000","last_became_peered":"2024-04-26T04:55:08.142745+0000","last_unstale":"2024-04-26T05:03:23.150394+0000","last_undegraded":"2024-04-26T05:03:23.150394+0000","last_fullsized":"2024-04-26T05:03:23.150394+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:08:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":1331,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575584+0000","last_change":"2024-04-26T04:56:12.731021+0000","last_active":"2024-04-26T05:03:25.575584+0000","last_peered":"2024-04-26T05:03:25.575584+0000","last_clean":"2024-04-26T05:03:25.575584+0000","last_became_active":"2024-04-26T04:56:12.730390+0000","last_became_peered":"2024-04-26T04:56:12.730390+0000","last_unstale":"2024-04-26T05:03:25.575584+0000","last_undegraded":"2024-04-26T05:03:25.575584+0000","last_fullsized":"2024-04-26T05:03:25.575584+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:00:15.121779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1270,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.849336+0000","last_change":"2024-04-26T04:56:48.329381+0000","last_active":"2024-04-26T05:03:25.849336+0000","last_peered":"2024-04-26T05:03:25.849336+0000","last_clean":"2024-04-26T05:03:25.849336+0000","last_became_active":"2024-04-26T04:56:48.329058+0000","last_became_peered":"2024-04-26T04:56:48.329058+0000","last_unstale":"2024-04-26T05:03:25.849336+0000","last_undegraded":"2024-04-26T05:03:25.849336+0000","last_fullsized":"2024-04-26T05:03:25.849336+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:49:40.848039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":1244,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844926+0000","last_change":"2024-04-26T04:56:48.339424+0000","last_active":"2024-04-26T05:03:25.844926+0000","last_peered":"2024-04-26T05:03:25.844926+0000","last_clean":"2024-04-26T05:03:25.844926+0000","last_became_active":"2024-04-26T04:56:48.333406+0000","last_became_peered":"2024-04-26T04:56:48.333406+0000","last_unstale":"2024-04-26T05:03:25.844926+0000","last_undegraded":"2024-04-26T05:03:25.844926+0000","last_fullsized":"2024-04-26T05:03:25.844926+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:03:03.160685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150808+0000","last_change":"2024-04-26T04:56:48.350816+0000","last_active":"2024-04-26T05:03:23.150808+0000","last_peered":"2024-04-26T05:03:23.150808+0000","last_clean":"2024-04-26T05:03:23.150808+0000","last_became_active":"2024-04-26T04:56:48.350216+0000","last_became_peered":"2024-04-26T04:56:48.350216+0000","last_unstale":"2024-04-26T05:03:23.150808+0000","last_undegraded":"2024-04-26T05:03:23.150808+0000","last_fullsized":"2024-04-26T05:03:23.150808+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:02:35.492294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":902,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120654+0000","last_change":"2024-04-26T04:45:21.634660+0000","last_active":"2024-04-26T05:03:23.120654+0000","last_peered":"2024-04-26T05:03:23.120654+0000","last_clean":"2024-04-26T05:03:23.120654+0000","last_became_active":"2024-04-26T04:45:21.634404+0000","last_became_peered":"2024-04-26T04:45:21.634404+0000","last_unstale":"2024-04-26T05:03:23.120654+0000","last_undegraded":"2024-04-26T05:03:23.120654+0000","last_fullsized":"2024-04-26T05:03:23.120654+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:28:20.390001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150627+0000","last_change":"2024-04-26T04:56:48.304826+0000","last_active":"2024-04-26T05:03:23.150627+0000","last_peered":"2024-04-26T05:03:23.150627+0000","last_clean":"2024-04-26T05:03:23.150627+0000","last_became_active":"2024-04-26T04:56:48.303286+0000","last_became_peered":"2024-04-26T04:56:48.303286+0000","last_unstale":"2024-04-26T05:03:23.150627+0000","last_undegraded":"2024-04-26T05:03:23.150627+0000","last_fullsized":"2024-04-26T05:03:23.150627+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:49:40.804289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":919,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120139+0000","last_change":"2024-04-26T04:55:08.152104+0000","last_active":"2024-04-26T05:03:23.120139+0000","last_peered":"2024-04-26T05:03:23.120139+0000","last_clean":"2024-04-26T05:03:23.120139+0000","last_became_active":"2024-04-26T04:55:08.151196+0000","last_became_peered":"2024-04-26T04:55:08.151196+0000","last_unstale":"2024-04-26T05:03:23.120139+0000","last_undegraded":"2024-04-26T05:03:23.120139+0000","last_fullsized":"2024-04-26T05:03:23.120139+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:55:37.905898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1263,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844565+0000","last_change":"2024-04-26T04:56:48.305950+0000","last_active":"2024-04-26T05:03:25.844565+0000","last_peered":"2024-04-26T05:03:25.844565+0000","last_clean":"2024-04-26T05:03:25.844565+0000","last_became_active":"2024-04-26T04:56:48.305276+0000","last_became_peered":"2024-04-26T04:56:48.305276+0000","last_unstale":"2024-04-26T05:03:25.844565+0000","last_undegraded":"2024-04-26T05:03:25.844565+0000","last_fullsized":"2024-04-26T05:03:25.844565+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:48.526947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":1318,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575077+0000","last_change":"2024-04-26T04:54:32.837927+0000","last_active":"2024-04-26T05:03:25.575077+0000","last_peered":"2024-04-26T05:03:25.575077+0000","last_clean":"2024-04-26T05:03:25.575077+0000","last_became_active":"2024-04-26T04:54:32.837301+0000","last_became_peered":"2024-04-26T04:54:32.837301+0000","last_unstale":"2024-04-26T05:03:25.575077+0000","last_undegraded":"2024-04-26T05:03:25.575077+0000","last_fullsized":"2024-04-26T05:03:25.575077+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:35:07.063171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":899,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150250+0000","last_change":"2024-04-26T04:45:22.644110+0000","last_active":"2024-04-26T05:03:23.150250+0000","last_peered":"2024-04-26T05:03:23.150250+0000","last_clean":"2024-04-26T05:03:23.150250+0000","last_became_active":"2024-04-26T04:45:22.643811+0000","last_became_peered":"2024-04-26T04:45:22.643811+0000","last_unstale":"2024-04-26T05:03:23.150250+0000","last_undegraded":"2024-04-26T05:03:23.150250+0000","last_fullsized":"2024-04-26T05:03:23.150250+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:20:05.112184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":1328,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574675+0000","last_change":"2024-04-26T04:55:08.147106+0000","last_active":"2024-04-26T05:03:25.574675+0000","last_peered":"2024-04-26T05:03:25.574675+0000","last_clean":"2024-04-26T05:03:25.574675+0000","last_became_active":"2024-04-26T04:55:08.146275+0000","last_became_peered":"2024-04-26T04:55:08.146275+0000","last_unstale":"2024-04-26T05:03:25.574675+0000","last_undegraded":"2024-04-26T05:03:25.574675+0000","last_fullsized":"2024-04-26T05:03:25.574675+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:11:07.229263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":919,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782816+0000","last_change":"2024-04-26T04:56:12.722240+0000","last_active":"2024-04-26T05:03:22.782816+0000","last_peered":"2024-04-26T05:03:22.782816+0000","last_clean":"2024-04-26T05:03:22.782816+0000","last_became_active":"2024-04-26T04:56:12.721910+0000","last_became_peered":"2024-04-26T04:56:12.721910+0000","last_unstale":"2024-04-26T05:03:22.782816+0000","last_undegraded":"2024-04-26T05:03:22.782816+0000","last_fullsized":"2024-04-26T05:03:22.782816+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:58:21.395885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1253,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843256+0000","last_change":"2024-04-26T04:56:48.342021+0000","last_active":"2024-04-26T05:03:25.843256+0000","last_peered":"2024-04-26T05:03:25.843256+0000","last_clean":"2024-04-26T05:03:25.843256+0000","last_became_active":"2024-04-26T04:56:48.321805+0000","last_became_peered":"2024-04-26T04:56:48.321805+0000","last_unstale":"2024-04-26T05:03:25.843256+0000","last_undegraded":"2024-04-26T05:03:25.843256+0000","last_fullsized":"2024-04-26T05:03:25.843256+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:59:57.950690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1290,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843945+0000","last_change":"2024-04-26T04:55:08.151058+0000","last_active":"2024-04-26T05:03:25.843945+0000","last_peered":"2024-04-26T05:03:25.843945+0000","last_clean":"2024-04-26T05:03:25.843945+0000","last_became_active":"2024-04-26T04:55:08.148964+0000","last_became_peered":"2024-04-26T04:55:08.148964+0000","last_unstale":"2024-04-26T05:03:25.843945+0000","last_undegraded":"2024-04-26T05:03:25.843945+0000","last_fullsized":"2024-04-26T05:03:25.843945+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:05:35.107253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1229,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843745+0000","last_change":"2024-04-26T04:56:48.319853+0000","last_active":"2024-04-26T05:03:25.843745+0000","last_peered":"2024-04-26T05:03:25.843745+0000","last_clean":"2024-04-26T05:03:25.843745+0000","last_became_active":"2024-04-26T04:56:48.317126+0000","last_became_peered":"2024-04-26T04:56:48.317126+0000","last_unstale":"2024-04-26T05:03:25.843745+0000","last_undegraded":"2024-04-26T05:03:25.843745+0000","last_fullsized":"2024-04-26T05:03:25.843745+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:33:58.920263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1316,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575000+0000","last_change":"2024-04-26T04:54:32.837808+0000","last_active":"2024-04-26T05:03:25.575000+0000","last_peered":"2024-04-26T05:03:25.575000+0000","last_clean":"2024-04-26T05:03:25.575000+0000","last_became_active":"2024-04-26T04:54:32.837209+0000","last_became_peered":"2024-04-26T04:54:32.837209+0000","last_unstale":"2024-04-26T05:03:25.575000+0000","last_undegraded":"2024-04-26T05:03:25.575000+0000","last_fullsized":"2024-04-26T05:03:25.575000+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:09:20.821127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1298,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844306+0000","last_change":"2024-04-26T04:56:12.737134+0000","last_active":"2024-04-26T05:03:25.844306+0000","last_peered":"2024-04-26T05:03:25.844306+0000","last_clean":"2024-04-26T05:03:25.844306+0000","last_became_active":"2024-04-26T04:56:12.736803+0000","last_became_peered":"2024-04-26T04:56:12.736803+0000","last_unstale":"2024-04-26T05:03:25.844306+0000","last_undegraded":"2024-04-26T05:03:25.844306+0000","last_fullsized":"2024-04-26T05:03:25.844306+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:19:42.676488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":919,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506134+0000","last_change":"2024-04-26T04:54:32.839765+0000","last_active":"2024-04-26T05:03:23.506134+0000","last_peered":"2024-04-26T05:03:23.506134+0000","last_clean":"2024-04-26T05:03:23.506134+0000","last_became_active":"2024-04-26T04:54:32.839373+0000","last_became_peered":"2024-04-26T04:54:32.839373+0000","last_unstale":"2024-04-26T05:03:23.506134+0000","last_undegraded":"2024-04-26T05:03:23.506134+0000","last_fullsized":"2024-04-26T05:03:23.506134+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:12.463770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1334,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574599+0000","last_change":"2024-04-26T04:56:12.731857+0000","last_active":"2024-04-26T05:03:25.574599+0000","last_peered":"2024-04-26T05:03:25.574599+0000","last_clean":"2024-04-26T05:03:25.574599+0000","last_became_active":"2024-04-26T04:56:12.730945+0000","last_became_peered":"2024-04-26T04:56:12.730945+0000","last_unstale":"2024-04-26T05:03:25.574599+0000","last_undegraded":"2024-04-26T05:03:25.574599+0000","last_fullsized":"2024-04-26T05:03:25.574599+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:10:59.157563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574379+0000","last_change":"2024-04-26T04:56:48.293829+0000","last_active":"2024-04-26T05:03:25.574379+0000","last_peered":"2024-04-26T05:03:25.574379+0000","last_clean":"2024-04-26T05:03:25.574379+0000","last_became_active":"2024-04-26T04:56:48.293466+0000","last_became_peered":"2024-04-26T04:56:48.293466+0000","last_unstale":"2024-04-26T05:03:25.574379+0000","last_undegraded":"2024-04-26T05:03:25.574379+0000","last_fullsized":"2024-04-26T05:03:25.574379+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:49:52.040900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":936,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.505587+0000","last_change":"2024-04-26T04:55:08.146516+0000","last_active":"2024-04-26T05:03:23.505587+0000","last_peered":"2024-04-26T05:03:23.505587+0000","last_clean":"2024-04-26T05:03:23.505587+0000","last_became_active":"2024-04-26T04:55:08.146202+0000","last_became_peered":"2024-04-26T04:55:08.146202+0000","last_unstale":"2024-04-26T05:03:23.505587+0000","last_undegraded":"2024-04-26T05:03:23.505587+0000","last_fullsized":"2024-04-26T05:03:23.505587+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:48:52.083414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":897,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782494+0000","last_change":"2024-04-26T04:45:22.645366+0000","last_active":"2024-04-26T05:03:22.782494+0000","last_peered":"2024-04-26T05:03:22.782494+0000","last_clean":"2024-04-26T05:03:22.782494+0000","last_became_active":"2024-04-26T04:45:22.645130+0000","last_became_peered":"2024-04-26T04:45:22.645130+0000","last_unstale":"2024-04-26T05:03:22.782494+0000","last_undegraded":"2024-04-26T05:03:22.782494+0000","last_fullsized":"2024-04-26T05:03:22.782494+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:52:52.129108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1208,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843237+0000","last_change":"2024-04-26T04:56:48.346494+0000","last_active":"2024-04-26T05:03:25.843237+0000","last_peered":"2024-04-26T05:03:25.843237+0000","last_clean":"2024-04-26T05:03:25.843237+0000","last_became_active":"2024-04-26T04:56:48.345671+0000","last_became_peered":"2024-04-26T04:56:48.345671+0000","last_unstale":"2024-04-26T05:03:25.843237+0000","last_undegraded":"2024-04-26T05:03:25.843237+0000","last_fullsized":"2024-04-26T05:03:25.843237+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:15:11.140297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"264'161","reported_seq":1296,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.505623+0000","last_change":"2024-04-26T04:54:32.846292+0000","last_active":"2024-04-26T05:03:23.505623+0000","last_peered":"2024-04-26T05:03:23.505623+0000","last_clean":"2024-04-26T05:03:23.505623+0000","last_became_active":"2024-04-26T04:54:32.845766+0000","last_became_peered":"2024-04-26T04:54:32.845766+0000","last_unstale":"2024-04-26T05:03:23.505623+0000","last_undegraded":"2024-04-26T05:03:23.505623+0000","last_fullsized":"2024-04-26T05:03:23.505623+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_clean_scrub_stamp":"2024-04-26T04:41:33.492763+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:03:40.959482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":921,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.119923+0000","last_change":"2024-04-26T04:56:12.719197+0000","last_active":"2024-04-26T05:03:23.119923+0000","last_peered":"2024-04-26T05:03:23.119923+0000","last_clean":"2024-04-26T05:03:23.119923+0000","last_became_active":"2024-04-26T04:56:12.718698+0000","last_became_peered":"2024-04-26T04:56:12.718698+0000","last_unstale":"2024-04-26T05:03:23.119923+0000","last_undegraded":"2024-04-26T05:03:23.119923+0000","last_fullsized":"2024-04-26T05:03:23.119923+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:14:19.216584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1307,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.846403+0000","last_change":"2024-04-26T04:55:08.153240+0000","last_active":"2024-04-26T05:03:25.846403+0000","last_peered":"2024-04-26T05:03:25.846403+0000","last_clean":"2024-04-26T05:03:25.846403+0000","last_became_active":"2024-04-26T04:55:08.152474+0000","last_became_peered":"2024-04-26T04:55:08.152474+0000","last_unstale":"2024-04-26T05:03:25.846403+0000","last_undegraded":"2024-04-26T05:03:25.846403+0000","last_fullsized":"2024-04-26T05:03:25.846403+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:42:17.914142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1298,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842864+0000","last_change":"2024-04-26T04:56:12.731955+0000","last_active":"2024-04-26T05:03:25.842864+0000","last_peered":"2024-04-26T05:03:25.842864+0000","last_clean":"2024-04-26T05:03:25.842864+0000","last_became_active":"2024-04-26T04:56:12.731201+0000","last_became_peered":"2024-04-26T04:56:12.731201+0000","last_unstale":"2024-04-26T05:03:25.842864+0000","last_undegraded":"2024-04-26T05:03:25.842864+0000","last_fullsized":"2024-04-26T05:03:25.842864+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:10:01.904844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1273,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844229+0000","last_change":"2024-04-26T04:56:48.327431+0000","last_active":"2024-04-26T05:03:25.844229+0000","last_peered":"2024-04-26T05:03:25.844229+0000","last_clean":"2024-04-26T05:03:25.844229+0000","last_became_active":"2024-04-26T04:56:48.327119+0000","last_became_peered":"2024-04-26T04:56:48.327119+0000","last_unstale":"2024-04-26T05:03:25.844229+0000","last_undegraded":"2024-04-26T05:03:25.844229+0000","last_fullsized":"2024-04-26T05:03:25.844229+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:36:31.137617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1308,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.845186+0000","last_change":"2024-04-26T04:56:48.329729+0000","last_active":"2024-04-26T05:03:25.845186+0000","last_peered":"2024-04-26T05:03:25.845186+0000","last_clean":"2024-04-26T05:03:25.845186+0000","last_became_active":"2024-04-26T04:56:48.329429+0000","last_became_peered":"2024-04-26T04:56:48.329429+0000","last_unstale":"2024-04-26T05:03:25.845186+0000","last_undegraded":"2024-04-26T05:03:25.845186+0000","last_fullsized":"2024-04-26T05:03:25.845186+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:19:08.924267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150807+0000","last_change":"2024-04-26T04:54:32.847179+0000","last_active":"2024-04-26T05:03:23.150807+0000","last_peered":"2024-04-26T05:03:23.150807+0000","last_clean":"2024-04-26T05:03:23.150807+0000","last_became_active":"2024-04-26T04:54:32.846868+0000","last_became_peered":"2024-04-26T04:54:32.846868+0000","last_unstale":"2024-04-26T05:03:23.150807+0000","last_undegraded":"2024-04-26T05:03:23.150807+0000","last_fullsized":"2024-04-26T05:03:23.150807+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:42:14.779856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":921,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.119918+0000","last_change":"2024-04-26T04:55:08.151594+0000","last_active":"2024-04-26T05:03:23.119918+0000","last_peered":"2024-04-26T05:03:23.119918+0000","last_clean":"2024-04-26T05:03:23.119918+0000","last_became_active":"2024-04-26T04:55:08.151004+0000","last_became_peered":"2024-04-26T04:55:08.151004+0000","last_unstale":"2024-04-26T05:03:23.119918+0000","last_undegraded":"2024-04-26T05:03:23.119918+0000","last_fullsized":"2024-04-26T05:03:23.119918+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:30:22.501053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1290,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843109+0000","last_change":"2024-04-26T04:55:08.148075+0000","last_active":"2024-04-26T05:03:25.843109+0000","last_peered":"2024-04-26T05:03:25.843109+0000","last_clean":"2024-04-26T05:03:25.843109+0000","last_became_active":"2024-04-26T04:55:08.145504+0000","last_became_peered":"2024-04-26T04:55:08.145504+0000","last_unstale":"2024-04-26T05:03:25.843109+0000","last_undegraded":"2024-04-26T05:03:25.843109+0000","last_fullsized":"2024-04-26T05:03:25.843109+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:47:21.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":934,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120529+0000","last_change":"2024-04-26T04:56:48.349013+0000","last_active":"2024-04-26T05:03:23.120529+0000","last_peered":"2024-04-26T05:03:23.120529+0000","last_clean":"2024-04-26T05:03:23.120529+0000","last_became_active":"2024-04-26T04:56:48.348529+0000","last_became_peered":"2024-04-26T04:56:48.348529+0000","last_unstale":"2024-04-26T05:03:23.120529+0000","last_undegraded":"2024-04-26T05:03:23.120529+0000","last_fullsized":"2024-04-26T05:03:23.120529+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:31:51.481703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1312,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844455+0000","last_change":"2024-04-26T04:55:08.148455+0000","last_active":"2024-04-26T05:03:25.844455+0000","last_peered":"2024-04-26T05:03:25.844455+0000","last_clean":"2024-04-26T05:03:25.844455+0000","last_became_active":"2024-04-26T04:55:08.144476+0000","last_became_peered":"2024-04-26T04:55:08.144476+0000","last_unstale":"2024-04-26T05:03:25.844455+0000","last_undegraded":"2024-04-26T05:03:25.844455+0000","last_fullsized":"2024-04-26T05:03:25.844455+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:39:01.385594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1299,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.846364+0000","last_change":"2024-04-26T04:56:48.312291+0000","last_active":"2024-04-26T05:03:25.846364+0000","last_peered":"2024-04-26T05:03:25.846364+0000","last_clean":"2024-04-26T05:03:25.846364+0000","last_became_active":"2024-04-26T04:56:48.311721+0000","last_became_peered":"2024-04-26T04:56:48.311721+0000","last_unstale":"2024-04-26T05:03:25.846364+0000","last_undegraded":"2024-04-26T05:03:25.846364+0000","last_fullsized":"2024-04-26T05:03:25.846364+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:48:45.381490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1220,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842135+0000","last_change":"2024-04-26T04:56:48.345721+0000","last_active":"2024-04-26T05:03:25.842135+0000","last_peered":"2024-04-26T05:03:25.842135+0000","last_clean":"2024-04-26T05:03:25.842135+0000","last_became_active":"2024-04-26T04:56:48.339533+0000","last_became_peered":"2024-04-26T04:56:48.339533+0000","last_unstale":"2024-04-26T05:03:25.842135+0000","last_undegraded":"2024-04-26T05:03:25.842135+0000","last_fullsized":"2024-04-26T05:03:25.842135+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:36:47.759686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.151127+0000","last_change":"2024-04-26T04:55:08.150897+0000","last_active":"2024-04-26T05:03:23.151127+0000","last_peered":"2024-04-26T05:03:23.151127+0000","last_clean":"2024-04-26T05:03:23.151127+0000","last_became_active":"2024-04-26T04:55:08.150583+0000","last_became_peered":"2024-04-26T04:55:08.150583+0000","last_unstale":"2024-04-26T05:03:23.151127+0000","last_undegraded":"2024-04-26T05:03:23.151127+0000","last_fullsized":"2024-04-26T05:03:23.151127+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:02:06.222606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150241+0000","last_change":"2024-04-26T04:56:12.711091+0000","last_active":"2024-04-26T05:03:23.150241+0000","last_peered":"2024-04-26T05:03:23.150241+0000","last_clean":"2024-04-26T05:03:23.150241+0000","last_became_active":"2024-04-26T04:56:12.710765+0000","last_became_peered":"2024-04-26T04:56:12.710765+0000","last_unstale":"2024-04-26T05:03:23.150241+0000","last_undegraded":"2024-04-26T05:03:23.150241+0000","last_fullsized":"2024-04-26T05:03:23.150241+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:42:31.933634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150800+0000","last_change":"2024-04-26T04:56:12.722586+0000","last_active":"2024-04-26T05:03:23.150800+0000","last_peered":"2024-04-26T05:03:23.150800+0000","last_clean":"2024-04-26T05:03:23.150800+0000","last_became_active":"2024-04-26T04:56:12.716370+0000","last_became_peered":"2024-04-26T04:56:12.716370+0000","last_unstale":"2024-04-26T05:03:23.150800+0000","last_undegraded":"2024-04-26T05:03:23.150800+0000","last_fullsized":"2024-04-26T05:03:23.150800+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:24:53.490574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":915,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782192+0000","last_change":"2024-04-26T04:55:08.149285+0000","last_active":"2024-04-26T05:03:22.782192+0000","last_peered":"2024-04-26T05:03:22.782192+0000","last_clean":"2024-04-26T05:03:22.782192+0000","last_became_active":"2024-04-26T04:55:08.148920+0000","last_became_peered":"2024-04-26T04:55:08.148920+0000","last_unstale":"2024-04-26T05:03:22.782192+0000","last_undegraded":"2024-04-26T05:03:22.782192+0000","last_fullsized":"2024-04-26T05:03:22.782192+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:54:20.877407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.151311+0000","last_change":"2024-04-26T04:56:48.306967+0000","last_active":"2024-04-26T05:03:23.151311+0000","last_peered":"2024-04-26T05:03:23.151311+0000","last_clean":"2024-04-26T05:03:23.151311+0000","last_became_active":"2024-04-26T04:56:48.306336+0000","last_became_peered":"2024-04-26T04:56:48.306336+0000","last_unstale":"2024-04-26T05:03:23.151311+0000","last_undegraded":"2024-04-26T05:03:23.151311+0000","last_fullsized":"2024-04-26T05:03:23.151311+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:43:53.876555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":1229,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.848544+0000","last_change":"2024-04-26T04:56:48.299738+0000","last_active":"2024-04-26T05:03:25.848544+0000","last_peered":"2024-04-26T05:03:25.848544+0000","last_clean":"2024-04-26T05:03:25.848544+0000","last_became_active":"2024-04-26T04:56:48.297286+0000","last_became_peered":"2024-04-26T04:56:48.297286+0000","last_unstale":"2024-04-26T05:03:25.848544+0000","last_undegraded":"2024-04-26T05:03:25.848544+0000","last_fullsized":"2024-04-26T05:03:25.848544+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:30:59.057125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574926+0000","last_change":"2024-04-26T04:55:08.146580+0000","last_active":"2024-04-26T05:03:25.574926+0000","last_peered":"2024-04-26T05:03:25.574926+0000","last_clean":"2024-04-26T05:03:25.574926+0000","last_became_active":"2024-04-26T04:55:08.146221+0000","last_became_peered":"2024-04-26T04:55:08.146221+0000","last_unstale":"2024-04-26T05:03:25.574926+0000","last_undegraded":"2024-04-26T05:03:25.574926+0000","last_fullsized":"2024-04-26T05:03:25.574926+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:05:30.269062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120224+0000","last_change":"2024-04-26T04:56:48.305114+0000","last_active":"2024-04-26T05:03:23.120224+0000","last_peered":"2024-04-26T05:03:23.120224+0000","last_clean":"2024-04-26T05:03:23.120224+0000","last_became_active":"2024-04-26T04:56:48.304596+0000","last_became_peered":"2024-04-26T04:56:48.304596+0000","last_unstale":"2024-04-26T05:03:23.120224+0000","last_undegraded":"2024-04-26T05:03:23.120224+0000","last_fullsized":"2024-04-26T05:03:23.120224+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:23:08.041144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":924,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782274+0000","last_change":"2024-04-26T04:56:12.719957+0000","last_active":"2024-04-26T05:03:22.782274+0000","last_peered":"2024-04-26T05:03:22.782274+0000","last_clean":"2024-04-26T05:03:22.782274+0000","last_became_active":"2024-04-26T04:56:12.719656+0000","last_became_peered":"2024-04-26T04:56:12.719656+0000","last_unstale":"2024-04-26T05:03:22.782274+0000","last_undegraded":"2024-04-26T05:03:22.782274+0000","last_fullsized":"2024-04-26T05:03:22.782274+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:04:08.753692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":899,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506176+0000","last_change":"2024-04-26T04:45:21.638699+0000","last_active":"2024-04-26T05:03:23.506176+0000","last_peered":"2024-04-26T05:03:23.506176+0000","last_clean":"2024-04-26T05:03:23.506176+0000","last_became_active":"2024-04-26T04:45:21.638390+0000","last_became_peered":"2024-04-26T04:45:21.638390+0000","last_unstale":"2024-04-26T05:03:23.506176+0000","last_undegraded":"2024-04-26T05:03:23.506176+0000","last_fullsized":"2024-04-26T05:03:23.506176+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:29:59.647984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150412+0000","last_change":"2024-04-26T04:56:48.303190+0000","last_active":"2024-04-26T05:03:23.150412+0000","last_peered":"2024-04-26T05:03:23.150412+0000","last_clean":"2024-04-26T05:03:23.150412+0000","last_became_active":"2024-04-26T04:56:48.302324+0000","last_became_peered":"2024-04-26T04:56:48.302324+0000","last_unstale":"2024-04-26T05:03:23.150412+0000","last_undegraded":"2024-04-26T05:03:23.150412+0000","last_fullsized":"2024-04-26T05:03:23.150412+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:38:28.077865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506153+0000","last_change":"2024-04-26T04:55:08.146674+0000","last_active":"2024-04-26T05:03:23.506153+0000","last_peered":"2024-04-26T05:03:23.506153+0000","last_clean":"2024-04-26T05:03:23.506153+0000","last_became_active":"2024-04-26T04:55:08.146280+0000","last_became_peered":"2024-04-26T04:55:08.146280+0000","last_unstale":"2024-04-26T05:03:23.506153+0000","last_undegraded":"2024-04-26T05:03:23.506153+0000","last_fullsized":"2024-04-26T05:03:23.506153+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:30:30.586691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":899,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120102+0000","last_change":"2024-04-26T04:45:22.642459+0000","last_active":"2024-04-26T05:03:23.120102+0000","last_peered":"2024-04-26T05:03:23.120102+0000","last_clean":"2024-04-26T05:03:23.120102+0000","last_became_active":"2024-04-26T04:45:22.642177+0000","last_became_peered":"2024-04-26T04:45:22.642177+0000","last_unstale":"2024-04-26T05:03:23.120102+0000","last_undegraded":"2024-04-26T05:03:23.120102+0000","last_fullsized":"2024-04-26T05:03:23.120102+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:14.494066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506327+0000","last_change":"2024-04-26T04:54:32.846236+0000","last_active":"2024-04-26T05:03:23.506327+0000","last_peered":"2024-04-26T05:03:23.506327+0000","last_clean":"2024-04-26T05:03:23.506327+0000","last_became_active":"2024-04-26T04:54:32.845837+0000","last_became_peered":"2024-04-26T04:54:32.845837+0000","last_unstale":"2024-04-26T05:03:23.506327+0000","last_undegraded":"2024-04-26T05:03:23.506327+0000","last_fullsized":"2024-04-26T05:03:23.506327+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:40:04.649128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1219,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843959+0000","last_change":"2024-04-26T04:56:48.325648+0000","last_active":"2024-04-26T05:03:25.843959+0000","last_peered":"2024-04-26T05:03:25.843959+0000","last_clean":"2024-04-26T05:03:25.843959+0000","last_became_active":"2024-04-26T04:56:48.320226+0000","last_became_peered":"2024-04-26T04:56:48.320226+0000","last_unstale":"2024-04-26T05:03:25.843959+0000","last_undegraded":"2024-04-26T05:03:25.843959+0000","last_fullsized":"2024-04-26T05:03:25.843959+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:39:30.071041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":923,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.781952+0000","last_change":"2024-04-26T04:54:32.835638+0000","last_active":"2024-04-26T05:03:22.781952+0000","last_peered":"2024-04-26T05:03:22.781952+0000","last_clean":"2024-04-26T05:03:22.781952+0000","last_became_active":"2024-04-26T04:54:32.835226+0000","last_became_peered":"2024-04-26T04:54:32.835226+0000","last_unstale":"2024-04-26T05:03:22.781952+0000","last_undegraded":"2024-04-26T05:03:22.781952+0000","last_fullsized":"2024-04-26T05:03:22.781952+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:12:14.882680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1324,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574505+0000","last_change":"2024-04-26T04:56:48.306018+0000","last_active":"2024-04-26T05:03:25.574505+0000","last_peered":"2024-04-26T05:03:25.574505+0000","last_clean":"2024-04-26T05:03:25.574505+0000","last_became_active":"2024-04-26T04:56:48.304492+0000","last_became_peered":"2024-04-26T04:56:48.304492+0000","last_unstale":"2024-04-26T05:03:25.574505+0000","last_undegraded":"2024-04-26T05:03:25.574505+0000","last_fullsized":"2024-04-26T05:03:25.574505+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:08:45.182964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":898,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.505712+0000","last_change":"2024-04-26T04:45:21.634430+0000","last_active":"2024-04-26T05:03:23.505712+0000","last_peered":"2024-04-26T05:03:23.505712+0000","last_clean":"2024-04-26T05:03:23.505712+0000","last_became_active":"2024-04-26T04:45:21.634149+0000","last_became_peered":"2024-04-26T04:45:21.634149+0000","last_unstale":"2024-04-26T05:03:23.505712+0000","last_undegraded":"2024-04-26T05:03:23.505712+0000","last_fullsized":"2024-04-26T05:03:23.505712+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:32.326287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120443+0000","last_change":"2024-04-26T04:56:48.349051+0000","last_active":"2024-04-26T05:03:23.120443+0000","last_peered":"2024-04-26T05:03:23.120443+0000","last_clean":"2024-04-26T05:03:23.120443+0000","last_became_active":"2024-04-26T04:56:48.348751+0000","last_became_peered":"2024-04-26T04:56:48.348751+0000","last_unstale":"2024-04-26T05:03:23.120443+0000","last_undegraded":"2024-04-26T05:03:23.120443+0000","last_fullsized":"2024-04-26T05:03:23.120443+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:26:16.122532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574340+0000","last_change":"2024-04-26T04:56:48.351609+0000","last_active":"2024-04-26T05:03:25.574340+0000","last_peered":"2024-04-26T05:03:25.574340+0000","last_clean":"2024-04-26T05:03:25.574340+0000","last_became_active":"2024-04-26T04:56:48.351148+0000","last_became_peered":"2024-04-26T04:56:48.351148+0000","last_unstale":"2024-04-26T05:03:25.574340+0000","last_undegraded":"2024-04-26T05:03:25.574340+0000","last_fullsized":"2024-04-26T05:03:25.574340+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:35:59.658257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1252,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843618+0000","last_change":"2024-04-26T04:56:48.345408+0000","last_active":"2024-04-26T05:03:25.843618+0000","last_peered":"2024-04-26T05:03:25.843618+0000","last_clean":"2024-04-26T05:03:25.843618+0000","last_became_active":"2024-04-26T04:56:48.339993+0000","last_became_peered":"2024-04-26T04:56:48.339993+0000","last_unstale":"2024-04-26T05:03:25.843618+0000","last_undegraded":"2024-04-26T05:03:25.843618+0000","last_fullsized":"2024-04-26T05:03:25.843618+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:13:34.643369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506210+0000","last_change":"2024-04-26T04:56:12.718233+0000","last_active":"2024-04-26T05:03:23.506210+0000","last_peered":"2024-04-26T05:03:23.506210+0000","last_clean":"2024-04-26T05:03:23.506210+0000","last_became_active":"2024-04-26T04:56:12.716681+0000","last_became_peered":"2024-04-26T04:56:12.716681+0000","last_unstale":"2024-04-26T05:03:23.506210+0000","last_undegraded":"2024-04-26T05:03:23.506210+0000","last_fullsized":"2024-04-26T05:03:23.506210+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:53:56.133301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150768+0000","last_change":"2024-04-26T04:55:08.150455+0000","last_active":"2024-04-26T05:03:23.150768+0000","last_peered":"2024-04-26T05:03:23.150768+0000","last_clean":"2024-04-26T05:03:23.150768+0000","last_became_active":"2024-04-26T04:55:08.149102+0000","last_became_peered":"2024-04-26T04:55:08.149102+0000","last_unstale":"2024-04-26T05:03:23.150768+0000","last_undegraded":"2024-04-26T05:03:23.150768+0000","last_fullsized":"2024-04-26T05:03:23.150768+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:27:55.353586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"346'22","reported_seq":922,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.151048+0000","last_change":"2024-04-26T04:45:21.631617+0000","last_active":"2024-04-26T05:03:23.151048+0000","last_peered":"2024-04-26T05:03:23.151048+0000","last_clean":"2024-04-26T05:03:23.151048+0000","last_became_active":"2024-04-26T04:45:21.631218+0000","last_became_peered":"2024-04-26T04:45:21.631218+0000","last_unstale":"2024-04-26T05:03:23.151048+0000","last_undegraded":"2024-04-26T05:03:23.151048+0000","last_fullsized":"2024-04-26T05:03:23.151048+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:50:12.166307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1298,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842881+0000","last_change":"2024-04-26T04:56:12.756700+0000","last_active":"2024-04-26T05:03:25.842881+0000","last_peered":"2024-04-26T05:03:25.842881+0000","last_clean":"2024-04-26T05:03:25.842881+0000","last_became_active":"2024-04-26T04:56:12.756373+0000","last_became_peered":"2024-04-26T04:56:12.756373+0000","last_unstale":"2024-04-26T05:03:25.842881+0000","last_undegraded":"2024-04-26T05:03:25.842881+0000","last_fullsized":"2024-04-26T05:03:25.842881+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:34:27.746415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":918,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506051+0000","last_change":"2024-04-26T04:56:48.334715+0000","last_active":"2024-04-26T05:03:23.506051+0000","last_peered":"2024-04-26T05:03:23.506051+0000","last_clean":"2024-04-26T05:03:23.506051+0000","last_became_active":"2024-04-26T04:56:48.333960+0000","last_became_peered":"2024-04-26T04:56:48.333960+0000","last_unstale":"2024-04-26T05:03:23.506051+0000","last_undegraded":"2024-04-26T05:03:23.506051+0000","last_fullsized":"2024-04-26T05:03:23.506051+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:25:42.094675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1208,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.849479+0000","last_change":"2024-04-26T04:56:48.314343+0000","last_active":"2024-04-26T05:03:25.849479+0000","last_peered":"2024-04-26T05:03:25.849479+0000","last_clean":"2024-04-26T05:03:25.849479+0000","last_became_active":"2024-04-26T04:56:48.313758+0000","last_became_peered":"2024-04-26T04:56:48.313758+0000","last_unstale":"2024-04-26T05:03:25.849479+0000","last_undegraded":"2024-04-26T05:03:25.849479+0000","last_fullsized":"2024-04-26T05:03:25.849479+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:53:43.804289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120261+0000","last_change":"2024-04-26T04:55:08.152158+0000","last_active":"2024-04-26T05:03:23.120261+0000","last_peered":"2024-04-26T05:03:23.120261+0000","last_clean":"2024-04-26T05:03:23.120261+0000","last_became_active":"2024-04-26T04:55:08.151105+0000","last_became_peered":"2024-04-26T04:55:08.151105+0000","last_unstale":"2024-04-26T05:03:23.120261+0000","last_undegraded":"2024-04-26T05:03:23.120261+0000","last_fullsized":"2024-04-26T05:03:23.120261+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:10:51.526015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1219,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843202+0000","last_change":"2024-04-26T04:56:48.347839+0000","last_active":"2024-04-26T05:03:25.843202+0000","last_peered":"2024-04-26T05:03:25.843202+0000","last_clean":"2024-04-26T05:03:25.843202+0000","last_became_active":"2024-04-26T04:56:48.340171+0000","last_became_peered":"2024-04-26T04:56:48.340171+0000","last_unstale":"2024-04-26T05:03:25.843202+0000","last_undegraded":"2024-04-26T05:03:25.843202+0000","last_fullsized":"2024-04-26T05:03:25.843202+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:07:44.332953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1324,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575422+0000","last_change":"2024-04-26T04:56:48.308107+0000","last_active":"2024-04-26T05:03:25.575422+0000","last_peered":"2024-04-26T05:03:25.575422+0000","last_clean":"2024-04-26T05:03:25.575422+0000","last_became_active":"2024-04-26T04:56:48.307386+0000","last_became_peered":"2024-04-26T04:56:48.307386+0000","last_unstale":"2024-04-26T05:03:25.575422+0000","last_undegraded":"2024-04-26T05:03:25.575422+0000","last_fullsized":"2024-04-26T05:03:25.575422+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:51:00.254568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":914,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782882+0000","last_change":"2024-04-26T04:55:08.149281+0000","last_active":"2024-04-26T05:03:22.782882+0000","last_peered":"2024-04-26T05:03:22.782882+0000","last_clean":"2024-04-26T05:03:22.782882+0000","last_became_active":"2024-04-26T04:55:08.148961+0000","last_became_peered":"2024-04-26T04:55:08.148961+0000","last_unstale":"2024-04-26T05:03:22.782882+0000","last_undegraded":"2024-04-26T05:03:22.782882+0000","last_fullsized":"2024-04-26T05:03:22.782882+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:12:31.364640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1237,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843400+0000","last_change":"2024-04-26T04:56:48.342262+0000","last_active":"2024-04-26T05:03:25.843400+0000","last_peered":"2024-04-26T05:03:25.843400+0000","last_clean":"2024-04-26T05:03:25.843400+0000","last_became_active":"2024-04-26T04:56:48.323634+0000","last_became_peered":"2024-04-26T04:56:48.323634+0000","last_unstale":"2024-04-26T05:03:25.843400+0000","last_undegraded":"2024-04-26T05:03:25.843400+0000","last_fullsized":"2024-04-26T05:03:25.843400+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:48:38.243275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574841+0000","last_change":"2024-04-26T04:56:48.306483+0000","last_active":"2024-04-26T05:03:25.574841+0000","last_peered":"2024-04-26T05:03:25.574841+0000","last_clean":"2024-04-26T05:03:25.574841+0000","last_became_active":"2024-04-26T04:56:48.304921+0000","last_became_peered":"2024-04-26T04:56:48.304921+0000","last_unstale":"2024-04-26T05:03:25.574841+0000","last_undegraded":"2024-04-26T05:03:25.574841+0000","last_fullsized":"2024-04-26T05:03:25.574841+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:04:58.167596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1297,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842271+0000","last_change":"2024-04-26T04:56:48.306726+0000","last_active":"2024-04-26T05:03:25.842271+0000","last_peered":"2024-04-26T05:03:25.842271+0000","last_clean":"2024-04-26T05:03:25.842271+0000","last_became_active":"2024-04-26T04:56:48.305823+0000","last_became_peered":"2024-04-26T04:56:48.305823+0000","last_unstale":"2024-04-26T05:03:25.842271+0000","last_undegraded":"2024-04-26T05:03:25.842271+0000","last_fullsized":"2024-04-26T05:03:25.842271+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:17:17.832270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1218,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.845847+0000","last_change":"2024-04-26T04:56:48.339586+0000","last_active":"2024-04-26T05:03:25.845847+0000","last_peered":"2024-04-26T05:03:25.845847+0000","last_clean":"2024-04-26T05:03:25.845847+0000","last_became_active":"2024-04-26T04:56:48.333394+0000","last_became_peered":"2024-04-26T04:56:48.333394+0000","last_unstale":"2024-04-26T05:03:25.845847+0000","last_undegraded":"2024-04-26T05:03:25.845847+0000","last_fullsized":"2024-04-26T05:03:25.845847+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:33:54.125128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":935,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150946+0000","last_change":"2024-04-26T04:56:12.725815+0000","last_active":"2024-04-26T05:03:23.150946+0000","last_peered":"2024-04-26T05:03:23.150946+0000","last_clean":"2024-04-26T05:03:23.150946+0000","last_became_active":"2024-04-26T04:56:12.725372+0000","last_became_peered":"2024-04-26T04:56:12.725372+0000","last_unstale":"2024-04-26T05:03:23.150946+0000","last_undegraded":"2024-04-26T05:03:23.150946+0000","last_fullsized":"2024-04-26T05:03:23.150946+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:01:20.219547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":954,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201531,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313384,"kb_used_data":16028,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93431768,"statfs":{"total":95995035648,"available":95674130432,"internally_reserved":0,"allocated":16412672,"data_stored":10366066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463168,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325652,"kb_used_data":16008,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93419500,"statfs":{"total":95995035648,"available":95661568000,"internally_reserved":0,"allocated":16392192,"data_stored":10365390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":553,"seq":2375116914798,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65220,"kb_used_data":15416,"kb_used_omap":55,"kb_used_meta":49736,"kb_avail":93679932,"statfs":{"total":95995035648,"available":95928250368,"internally_reserved":0,"allocated":15785984,"data_stored":9770509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":57122,"internal_metadata":50929886},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":519,"seq":2229088026741,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":70100,"kb_used_data":16004,"kb_used_omap":62,"kb_used_meta":54017,"kb_avail":93675052,"statfs":{"total":95995035648,"available":95923253248,"internally_reserved":0,"allocated":16388096,"data_stored":10361469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":63539,"internal_metadata":55314381},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":618,"seq":2654289789025,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65484,"kb_used_data":15424,"kb_used_omap":74,"kb_used_meta":49973,"kb_avail":93679668,"statfs":{"total":95995035648,"available":95927980032,"internally_reserved":0,"allocated":15794176,"data_stored":9771105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76149,"internal_metadata":51173003},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":636,"seq":2731599200346,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":61324,"kb_used_data":15424,"kb_used_omap":65,"kb_used_meta":45822,"kb_avail":93683828,"statfs":{"total":95995035648,"available":95932239872,"internally_reserved":0,"allocated":15794176,"data_stored":9770848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67394,"internal_metadata":46921918},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986440,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316892,"kb_used_data":15440,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93428260,"statfs":{"total":95995035648,"available":95670538240,"internally_reserved":0,"allocated":15810560,"data_stored":9775363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724805,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329164,"kb_used_data":15428,"kb_used_omap":1,"kb_used_meta":313726,"kb_avail":93415988,"statfs":{"total":95995035648,"available":95657971712,"internally_reserved":0,"allocated":15798272,"data_stored":9771378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321255768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":954,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:04:11.722 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:04:11.987 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:12.269 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:12 smithi111 ceph-mon[86360]: pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:12.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:12 smithi111 ceph-mon[86360]: from='client.35058 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:04:12.355 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:12 smithi005 ceph-mon[91801]: pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:12.355 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:12 smithi005 ceph-mon[91801]: from='client.35058 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:04:12.356 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:12 smithi005 ceph-mon[95809]: pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:12.356 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:12 smithi005 ceph-mon[95809]: from='client.35058 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:04:13.367 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:04:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:04:13] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:04:13.827 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:13.829 INFO:teuthology.orchestra.run.smithi005.stderr:dumped all 2024-04-26T05:04:14.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:14 smithi005 ceph-mon[91801]: pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:14.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:14 smithi005 ceph-mon[95809]: pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:14.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:14 smithi111 ceph-mon[86360]: pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:14.462 INFO:teuthology.orchestra.run.smithi005.stdout:{"pg_ready":true,"pg_map":{"version":1162,"stamp":"2024-04-26T05:04:12.567145+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1547220,"kb_used_data":125172,"kb_used_omap":264,"kb_used_meta":1421687,"kb_avail":748413996,"statfs":{"total":767960285184,"available":766375931904,"internally_reserved":0,"allocated":128176128,"data_stored":79952128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":271020,"internal_metadata":1455807828},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003219"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150394+0000","last_change":"2024-04-26T04:55:08.143144+0000","last_active":"2024-04-26T05:03:23.150394+0000","last_peered":"2024-04-26T05:03:23.150394+0000","last_clean":"2024-04-26T05:03:23.150394+0000","last_became_active":"2024-04-26T04:55:08.142745+0000","last_became_peered":"2024-04-26T04:55:08.142745+0000","last_unstale":"2024-04-26T05:03:23.150394+0000","last_undegraded":"2024-04-26T05:03:23.150394+0000","last_fullsized":"2024-04-26T05:03:23.150394+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:08:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":1331,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575584+0000","last_change":"2024-04-26T04:56:12.731021+0000","last_active":"2024-04-26T05:03:25.575584+0000","last_peered":"2024-04-26T05:03:25.575584+0000","last_clean":"2024-04-26T05:03:25.575584+0000","last_became_active":"2024-04-26T04:56:12.730390+0000","last_became_peered":"2024-04-26T04:56:12.730390+0000","last_unstale":"2024-04-26T05:03:25.575584+0000","last_undegraded":"2024-04-26T05:03:25.575584+0000","last_fullsized":"2024-04-26T05:03:25.575584+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:00:15.121779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1270,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.849336+0000","last_change":"2024-04-26T04:56:48.329381+0000","last_active":"2024-04-26T05:03:25.849336+0000","last_peered":"2024-04-26T05:03:25.849336+0000","last_clean":"2024-04-26T05:03:25.849336+0000","last_became_active":"2024-04-26T04:56:48.329058+0000","last_became_peered":"2024-04-26T04:56:48.329058+0000","last_unstale":"2024-04-26T05:03:25.849336+0000","last_undegraded":"2024-04-26T05:03:25.849336+0000","last_fullsized":"2024-04-26T05:03:25.849336+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:49:40.848039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":1244,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844926+0000","last_change":"2024-04-26T04:56:48.339424+0000","last_active":"2024-04-26T05:03:25.844926+0000","last_peered":"2024-04-26T05:03:25.844926+0000","last_clean":"2024-04-26T05:03:25.844926+0000","last_became_active":"2024-04-26T04:56:48.333406+0000","last_became_peered":"2024-04-26T04:56:48.333406+0000","last_unstale":"2024-04-26T05:03:25.844926+0000","last_undegraded":"2024-04-26T05:03:25.844926+0000","last_fullsized":"2024-04-26T05:03:25.844926+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:03:03.160685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150808+0000","last_change":"2024-04-26T04:56:48.350816+0000","last_active":"2024-04-26T05:03:23.150808+0000","last_peered":"2024-04-26T05:03:23.150808+0000","last_clean":"2024-04-26T05:03:23.150808+0000","last_became_active":"2024-04-26T04:56:48.350216+0000","last_became_peered":"2024-04-26T04:56:48.350216+0000","last_unstale":"2024-04-26T05:03:23.150808+0000","last_undegraded":"2024-04-26T05:03:23.150808+0000","last_fullsized":"2024-04-26T05:03:23.150808+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:02:35.492294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":902,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120654+0000","last_change":"2024-04-26T04:45:21.634660+0000","last_active":"2024-04-26T05:03:23.120654+0000","last_peered":"2024-04-26T05:03:23.120654+0000","last_clean":"2024-04-26T05:03:23.120654+0000","last_became_active":"2024-04-26T04:45:21.634404+0000","last_became_peered":"2024-04-26T04:45:21.634404+0000","last_unstale":"2024-04-26T05:03:23.120654+0000","last_undegraded":"2024-04-26T05:03:23.120654+0000","last_fullsized":"2024-04-26T05:03:23.120654+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:28:20.390001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150627+0000","last_change":"2024-04-26T04:56:48.304826+0000","last_active":"2024-04-26T05:03:23.150627+0000","last_peered":"2024-04-26T05:03:23.150627+0000","last_clean":"2024-04-26T05:03:23.150627+0000","last_became_active":"2024-04-26T04:56:48.303286+0000","last_became_peered":"2024-04-26T04:56:48.303286+0000","last_unstale":"2024-04-26T05:03:23.150627+0000","last_undegraded":"2024-04-26T05:03:23.150627+0000","last_fullsized":"2024-04-26T05:03:23.150627+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:49:40.804289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":919,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120139+0000","last_change":"2024-04-26T04:55:08.152104+0000","last_active":"2024-04-26T05:03:23.120139+0000","last_peered":"2024-04-26T05:03:23.120139+0000","last_clean":"2024-04-26T05:03:23.120139+0000","last_became_active":"2024-04-26T04:55:08.151196+0000","last_became_peered":"2024-04-26T04:55:08.151196+0000","last_unstale":"2024-04-26T05:03:23.120139+0000","last_undegraded":"2024-04-26T05:03:23.120139+0000","last_fullsized":"2024-04-26T05:03:23.120139+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:55:37.905898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1263,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844565+0000","last_change":"2024-04-26T04:56:48.305950+0000","last_active":"2024-04-26T05:03:25.844565+0000","last_peered":"2024-04-26T05:03:25.844565+0000","last_clean":"2024-04-26T05:03:25.844565+0000","last_became_active":"2024-04-26T04:56:48.305276+0000","last_became_peered":"2024-04-26T04:56:48.305276+0000","last_unstale":"2024-04-26T05:03:25.844565+0000","last_undegraded":"2024-04-26T05:03:25.844565+0000","last_fullsized":"2024-04-26T05:03:25.844565+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:48.526947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":1318,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575077+0000","last_change":"2024-04-26T04:54:32.837927+0000","last_active":"2024-04-26T05:03:25.575077+0000","last_peered":"2024-04-26T05:03:25.575077+0000","last_clean":"2024-04-26T05:03:25.575077+0000","last_became_active":"2024-04-26T04:54:32.837301+0000","last_became_peered":"2024-04-26T04:54:32.837301+0000","last_unstale":"2024-04-26T05:03:25.575077+0000","last_undegraded":"2024-04-26T05:03:25.575077+0000","last_fullsized":"2024-04-26T05:03:25.575077+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:35:07.063171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":899,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150250+0000","last_change":"2024-04-26T04:45:22.644110+0000","last_active":"2024-04-26T05:03:23.150250+0000","last_peered":"2024-04-26T05:03:23.150250+0000","last_clean":"2024-04-26T05:03:23.150250+0000","last_became_active":"2024-04-26T04:45:22.643811+0000","last_became_peered":"2024-04-26T04:45:22.643811+0000","last_unstale":"2024-04-26T05:03:23.150250+0000","last_undegraded":"2024-04-26T05:03:23.150250+0000","last_fullsized":"2024-04-26T05:03:23.150250+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:20:05.112184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":1328,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574675+0000","last_change":"2024-04-26T04:55:08.147106+0000","last_active":"2024-04-26T05:03:25.574675+0000","last_peered":"2024-04-26T05:03:25.574675+0000","last_clean":"2024-04-26T05:03:25.574675+0000","last_became_active":"2024-04-26T04:55:08.146275+0000","last_became_peered":"2024-04-26T04:55:08.146275+0000","last_unstale":"2024-04-26T05:03:25.574675+0000","last_undegraded":"2024-04-26T05:03:25.574675+0000","last_fullsized":"2024-04-26T05:03:25.574675+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:11:07.229263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":919,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782816+0000","last_change":"2024-04-26T04:56:12.722240+0000","last_active":"2024-04-26T05:03:22.782816+0000","last_peered":"2024-04-26T05:03:22.782816+0000","last_clean":"2024-04-26T05:03:22.782816+0000","last_became_active":"2024-04-26T04:56:12.721910+0000","last_became_peered":"2024-04-26T04:56:12.721910+0000","last_unstale":"2024-04-26T05:03:22.782816+0000","last_undegraded":"2024-04-26T05:03:22.782816+0000","last_fullsized":"2024-04-26T05:03:22.782816+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:58:21.395885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1253,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843256+0000","last_change":"2024-04-26T04:56:48.342021+0000","last_active":"2024-04-26T05:03:25.843256+0000","last_peered":"2024-04-26T05:03:25.843256+0000","last_clean":"2024-04-26T05:03:25.843256+0000","last_became_active":"2024-04-26T04:56:48.321805+0000","last_became_peered":"2024-04-26T04:56:48.321805+0000","last_unstale":"2024-04-26T05:03:25.843256+0000","last_undegraded":"2024-04-26T05:03:25.843256+0000","last_fullsized":"2024-04-26T05:03:25.843256+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:59:57.950690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1290,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843945+0000","last_change":"2024-04-26T04:55:08.151058+0000","last_active":"2024-04-26T05:03:25.843945+0000","last_peered":"2024-04-26T05:03:25.843945+0000","last_clean":"2024-04-26T05:03:25.843945+0000","last_became_active":"2024-04-26T04:55:08.148964+0000","last_became_peered":"2024-04-26T04:55:08.148964+0000","last_unstale":"2024-04-26T05:03:25.843945+0000","last_undegraded":"2024-04-26T05:03:25.843945+0000","last_fullsized":"2024-04-26T05:03:25.843945+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:05:35.107253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1229,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843745+0000","last_change":"2024-04-26T04:56:48.319853+0000","last_active":"2024-04-26T05:03:25.843745+0000","last_peered":"2024-04-26T05:03:25.843745+0000","last_clean":"2024-04-26T05:03:25.843745+0000","last_became_active":"2024-04-26T04:56:48.317126+0000","last_became_peered":"2024-04-26T04:56:48.317126+0000","last_unstale":"2024-04-26T05:03:25.843745+0000","last_undegraded":"2024-04-26T05:03:25.843745+0000","last_fullsized":"2024-04-26T05:03:25.843745+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:33:58.920263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1316,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575000+0000","last_change":"2024-04-26T04:54:32.837808+0000","last_active":"2024-04-26T05:03:25.575000+0000","last_peered":"2024-04-26T05:03:25.575000+0000","last_clean":"2024-04-26T05:03:25.575000+0000","last_became_active":"2024-04-26T04:54:32.837209+0000","last_became_peered":"2024-04-26T04:54:32.837209+0000","last_unstale":"2024-04-26T05:03:25.575000+0000","last_undegraded":"2024-04-26T05:03:25.575000+0000","last_fullsized":"2024-04-26T05:03:25.575000+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:09:20.821127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1298,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844306+0000","last_change":"2024-04-26T04:56:12.737134+0000","last_active":"2024-04-26T05:03:25.844306+0000","last_peered":"2024-04-26T05:03:25.844306+0000","last_clean":"2024-04-26T05:03:25.844306+0000","last_became_active":"2024-04-26T04:56:12.736803+0000","last_became_peered":"2024-04-26T04:56:12.736803+0000","last_unstale":"2024-04-26T05:03:25.844306+0000","last_undegraded":"2024-04-26T05:03:25.844306+0000","last_fullsized":"2024-04-26T05:03:25.844306+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:19:42.676488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":919,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506134+0000","last_change":"2024-04-26T04:54:32.839765+0000","last_active":"2024-04-26T05:03:23.506134+0000","last_peered":"2024-04-26T05:03:23.506134+0000","last_clean":"2024-04-26T05:03:23.506134+0000","last_became_active":"2024-04-26T04:54:32.839373+0000","last_became_peered":"2024-04-26T04:54:32.839373+0000","last_unstale":"2024-04-26T05:03:23.506134+0000","last_undegraded":"2024-04-26T05:03:23.506134+0000","last_fullsized":"2024-04-26T05:03:23.506134+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:12.463770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1334,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574599+0000","last_change":"2024-04-26T04:56:12.731857+0000","last_active":"2024-04-26T05:03:25.574599+0000","last_peered":"2024-04-26T05:03:25.574599+0000","last_clean":"2024-04-26T05:03:25.574599+0000","last_became_active":"2024-04-26T04:56:12.730945+0000","last_became_peered":"2024-04-26T04:56:12.730945+0000","last_unstale":"2024-04-26T05:03:25.574599+0000","last_undegraded":"2024-04-26T05:03:25.574599+0000","last_fullsized":"2024-04-26T05:03:25.574599+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:10:59.157563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574379+0000","last_change":"2024-04-26T04:56:48.293829+0000","last_active":"2024-04-26T05:03:25.574379+0000","last_peered":"2024-04-26T05:03:25.574379+0000","last_clean":"2024-04-26T05:03:25.574379+0000","last_became_active":"2024-04-26T04:56:48.293466+0000","last_became_peered":"2024-04-26T04:56:48.293466+0000","last_unstale":"2024-04-26T05:03:25.574379+0000","last_undegraded":"2024-04-26T05:03:25.574379+0000","last_fullsized":"2024-04-26T05:03:25.574379+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:49:52.040900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":936,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.505587+0000","last_change":"2024-04-26T04:55:08.146516+0000","last_active":"2024-04-26T05:03:23.505587+0000","last_peered":"2024-04-26T05:03:23.505587+0000","last_clean":"2024-04-26T05:03:23.505587+0000","last_became_active":"2024-04-26T04:55:08.146202+0000","last_became_peered":"2024-04-26T04:55:08.146202+0000","last_unstale":"2024-04-26T05:03:23.505587+0000","last_undegraded":"2024-04-26T05:03:23.505587+0000","last_fullsized":"2024-04-26T05:03:23.505587+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:48:52.083414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":897,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782494+0000","last_change":"2024-04-26T04:45:22.645366+0000","last_active":"2024-04-26T05:03:22.782494+0000","last_peered":"2024-04-26T05:03:22.782494+0000","last_clean":"2024-04-26T05:03:22.782494+0000","last_became_active":"2024-04-26T04:45:22.645130+0000","last_became_peered":"2024-04-26T04:45:22.645130+0000","last_unstale":"2024-04-26T05:03:22.782494+0000","last_undegraded":"2024-04-26T05:03:22.782494+0000","last_fullsized":"2024-04-26T05:03:22.782494+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:52:52.129108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1208,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843237+0000","last_change":"2024-04-26T04:56:48.346494+0000","last_active":"2024-04-26T05:03:25.843237+0000","last_peered":"2024-04-26T05:03:25.843237+0000","last_clean":"2024-04-26T05:03:25.843237+0000","last_became_active":"2024-04-26T04:56:48.345671+0000","last_became_peered":"2024-04-26T04:56:48.345671+0000","last_unstale":"2024-04-26T05:03:25.843237+0000","last_undegraded":"2024-04-26T05:03:25.843237+0000","last_fullsized":"2024-04-26T05:03:25.843237+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:15:11.140297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"264'161","reported_seq":1296,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.505623+0000","last_change":"2024-04-26T04:54:32.846292+0000","last_active":"2024-04-26T05:03:23.505623+0000","last_peered":"2024-04-26T05:03:23.505623+0000","last_clean":"2024-04-26T05:03:23.505623+0000","last_became_active":"2024-04-26T04:54:32.845766+0000","last_became_peered":"2024-04-26T04:54:32.845766+0000","last_unstale":"2024-04-26T05:03:23.505623+0000","last_undegraded":"2024-04-26T05:03:23.505623+0000","last_fullsized":"2024-04-26T05:03:23.505623+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:41:33.492763+0000","last_clean_scrub_stamp":"2024-04-26T04:41:33.492763+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:03:40.959482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":921,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.119923+0000","last_change":"2024-04-26T04:56:12.719197+0000","last_active":"2024-04-26T05:03:23.119923+0000","last_peered":"2024-04-26T05:03:23.119923+0000","last_clean":"2024-04-26T05:03:23.119923+0000","last_became_active":"2024-04-26T04:56:12.718698+0000","last_became_peered":"2024-04-26T04:56:12.718698+0000","last_unstale":"2024-04-26T05:03:23.119923+0000","last_undegraded":"2024-04-26T05:03:23.119923+0000","last_fullsized":"2024-04-26T05:03:23.119923+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:14:19.216584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1307,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.846403+0000","last_change":"2024-04-26T04:55:08.153240+0000","last_active":"2024-04-26T05:03:25.846403+0000","last_peered":"2024-04-26T05:03:25.846403+0000","last_clean":"2024-04-26T05:03:25.846403+0000","last_became_active":"2024-04-26T04:55:08.152474+0000","last_became_peered":"2024-04-26T04:55:08.152474+0000","last_unstale":"2024-04-26T05:03:25.846403+0000","last_undegraded":"2024-04-26T05:03:25.846403+0000","last_fullsized":"2024-04-26T05:03:25.846403+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:42:17.914142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1298,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842864+0000","last_change":"2024-04-26T04:56:12.731955+0000","last_active":"2024-04-26T05:03:25.842864+0000","last_peered":"2024-04-26T05:03:25.842864+0000","last_clean":"2024-04-26T05:03:25.842864+0000","last_became_active":"2024-04-26T04:56:12.731201+0000","last_became_peered":"2024-04-26T04:56:12.731201+0000","last_unstale":"2024-04-26T05:03:25.842864+0000","last_undegraded":"2024-04-26T05:03:25.842864+0000","last_fullsized":"2024-04-26T05:03:25.842864+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:10:01.904844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1273,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844229+0000","last_change":"2024-04-26T04:56:48.327431+0000","last_active":"2024-04-26T05:03:25.844229+0000","last_peered":"2024-04-26T05:03:25.844229+0000","last_clean":"2024-04-26T05:03:25.844229+0000","last_became_active":"2024-04-26T04:56:48.327119+0000","last_became_peered":"2024-04-26T04:56:48.327119+0000","last_unstale":"2024-04-26T05:03:25.844229+0000","last_undegraded":"2024-04-26T05:03:25.844229+0000","last_fullsized":"2024-04-26T05:03:25.844229+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:36:31.137617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1308,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.845186+0000","last_change":"2024-04-26T04:56:48.329729+0000","last_active":"2024-04-26T05:03:25.845186+0000","last_peered":"2024-04-26T05:03:25.845186+0000","last_clean":"2024-04-26T05:03:25.845186+0000","last_became_active":"2024-04-26T04:56:48.329429+0000","last_became_peered":"2024-04-26T04:56:48.329429+0000","last_unstale":"2024-04-26T05:03:25.845186+0000","last_undegraded":"2024-04-26T05:03:25.845186+0000","last_fullsized":"2024-04-26T05:03:25.845186+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:19:08.924267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150807+0000","last_change":"2024-04-26T04:54:32.847179+0000","last_active":"2024-04-26T05:03:23.150807+0000","last_peered":"2024-04-26T05:03:23.150807+0000","last_clean":"2024-04-26T05:03:23.150807+0000","last_became_active":"2024-04-26T04:54:32.846868+0000","last_became_peered":"2024-04-26T04:54:32.846868+0000","last_unstale":"2024-04-26T05:03:23.150807+0000","last_undegraded":"2024-04-26T05:03:23.150807+0000","last_fullsized":"2024-04-26T05:03:23.150807+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:42:14.779856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":921,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.119918+0000","last_change":"2024-04-26T04:55:08.151594+0000","last_active":"2024-04-26T05:03:23.119918+0000","last_peered":"2024-04-26T05:03:23.119918+0000","last_clean":"2024-04-26T05:03:23.119918+0000","last_became_active":"2024-04-26T04:55:08.151004+0000","last_became_peered":"2024-04-26T04:55:08.151004+0000","last_unstale":"2024-04-26T05:03:23.119918+0000","last_undegraded":"2024-04-26T05:03:23.119918+0000","last_fullsized":"2024-04-26T05:03:23.119918+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:30:22.501053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1290,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843109+0000","last_change":"2024-04-26T04:55:08.148075+0000","last_active":"2024-04-26T05:03:25.843109+0000","last_peered":"2024-04-26T05:03:25.843109+0000","last_clean":"2024-04-26T05:03:25.843109+0000","last_became_active":"2024-04-26T04:55:08.145504+0000","last_became_peered":"2024-04-26T04:55:08.145504+0000","last_unstale":"2024-04-26T05:03:25.843109+0000","last_undegraded":"2024-04-26T05:03:25.843109+0000","last_fullsized":"2024-04-26T05:03:25.843109+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:47:21.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":934,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120529+0000","last_change":"2024-04-26T04:56:48.349013+0000","last_active":"2024-04-26T05:03:23.120529+0000","last_peered":"2024-04-26T05:03:23.120529+0000","last_clean":"2024-04-26T05:03:23.120529+0000","last_became_active":"2024-04-26T04:56:48.348529+0000","last_became_peered":"2024-04-26T04:56:48.348529+0000","last_unstale":"2024-04-26T05:03:23.120529+0000","last_undegraded":"2024-04-26T05:03:23.120529+0000","last_fullsized":"2024-04-26T05:03:23.120529+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:31:51.481703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1312,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.844455+0000","last_change":"2024-04-26T04:55:08.148455+0000","last_active":"2024-04-26T05:03:25.844455+0000","last_peered":"2024-04-26T05:03:25.844455+0000","last_clean":"2024-04-26T05:03:25.844455+0000","last_became_active":"2024-04-26T04:55:08.144476+0000","last_became_peered":"2024-04-26T04:55:08.144476+0000","last_unstale":"2024-04-26T05:03:25.844455+0000","last_undegraded":"2024-04-26T05:03:25.844455+0000","last_fullsized":"2024-04-26T05:03:25.844455+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:44:07.467110+0000","last_clean_scrub_stamp":"2024-04-26T04:44:07.467110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:39:01.385594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1299,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.846364+0000","last_change":"2024-04-26T04:56:48.312291+0000","last_active":"2024-04-26T05:03:25.846364+0000","last_peered":"2024-04-26T05:03:25.846364+0000","last_clean":"2024-04-26T05:03:25.846364+0000","last_became_active":"2024-04-26T04:56:48.311721+0000","last_became_peered":"2024-04-26T04:56:48.311721+0000","last_unstale":"2024-04-26T05:03:25.846364+0000","last_undegraded":"2024-04-26T05:03:25.846364+0000","last_fullsized":"2024-04-26T05:03:25.846364+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:48:45.381490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1220,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842135+0000","last_change":"2024-04-26T04:56:48.345721+0000","last_active":"2024-04-26T05:03:25.842135+0000","last_peered":"2024-04-26T05:03:25.842135+0000","last_clean":"2024-04-26T05:03:25.842135+0000","last_became_active":"2024-04-26T04:56:48.339533+0000","last_became_peered":"2024-04-26T04:56:48.339533+0000","last_unstale":"2024-04-26T05:03:25.842135+0000","last_undegraded":"2024-04-26T05:03:25.842135+0000","last_fullsized":"2024-04-26T05:03:25.842135+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:36:47.759686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.151127+0000","last_change":"2024-04-26T04:55:08.150897+0000","last_active":"2024-04-26T05:03:23.151127+0000","last_peered":"2024-04-26T05:03:23.151127+0000","last_clean":"2024-04-26T05:03:23.151127+0000","last_became_active":"2024-04-26T04:55:08.150583+0000","last_became_peered":"2024-04-26T04:55:08.150583+0000","last_unstale":"2024-04-26T05:03:23.151127+0000","last_undegraded":"2024-04-26T05:03:23.151127+0000","last_fullsized":"2024-04-26T05:03:23.151127+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:02:06.222606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150241+0000","last_change":"2024-04-26T04:56:12.711091+0000","last_active":"2024-04-26T05:03:23.150241+0000","last_peered":"2024-04-26T05:03:23.150241+0000","last_clean":"2024-04-26T05:03:23.150241+0000","last_became_active":"2024-04-26T04:56:12.710765+0000","last_became_peered":"2024-04-26T04:56:12.710765+0000","last_unstale":"2024-04-26T05:03:23.150241+0000","last_undegraded":"2024-04-26T05:03:23.150241+0000","last_fullsized":"2024-04-26T05:03:23.150241+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:42:31.933634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150800+0000","last_change":"2024-04-26T04:56:12.722586+0000","last_active":"2024-04-26T05:03:23.150800+0000","last_peered":"2024-04-26T05:03:23.150800+0000","last_clean":"2024-04-26T05:03:23.150800+0000","last_became_active":"2024-04-26T04:56:12.716370+0000","last_became_peered":"2024-04-26T04:56:12.716370+0000","last_unstale":"2024-04-26T05:03:23.150800+0000","last_undegraded":"2024-04-26T05:03:23.150800+0000","last_fullsized":"2024-04-26T05:03:23.150800+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:24:53.490574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":915,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782192+0000","last_change":"2024-04-26T04:55:08.149285+0000","last_active":"2024-04-26T05:03:22.782192+0000","last_peered":"2024-04-26T05:03:22.782192+0000","last_clean":"2024-04-26T05:03:22.782192+0000","last_became_active":"2024-04-26T04:55:08.148920+0000","last_became_peered":"2024-04-26T04:55:08.148920+0000","last_unstale":"2024-04-26T05:03:22.782192+0000","last_undegraded":"2024-04-26T05:03:22.782192+0000","last_fullsized":"2024-04-26T05:03:22.782192+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:54:20.877407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.151311+0000","last_change":"2024-04-26T04:56:48.306967+0000","last_active":"2024-04-26T05:03:23.151311+0000","last_peered":"2024-04-26T05:03:23.151311+0000","last_clean":"2024-04-26T05:03:23.151311+0000","last_became_active":"2024-04-26T04:56:48.306336+0000","last_became_peered":"2024-04-26T04:56:48.306336+0000","last_unstale":"2024-04-26T05:03:23.151311+0000","last_undegraded":"2024-04-26T05:03:23.151311+0000","last_fullsized":"2024-04-26T05:03:23.151311+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:43:53.876555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":1229,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.848544+0000","last_change":"2024-04-26T04:56:48.299738+0000","last_active":"2024-04-26T05:03:25.848544+0000","last_peered":"2024-04-26T05:03:25.848544+0000","last_clean":"2024-04-26T05:03:25.848544+0000","last_became_active":"2024-04-26T04:56:48.297286+0000","last_became_peered":"2024-04-26T04:56:48.297286+0000","last_unstale":"2024-04-26T05:03:25.848544+0000","last_undegraded":"2024-04-26T05:03:25.848544+0000","last_fullsized":"2024-04-26T05:03:25.848544+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:30:59.057125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574926+0000","last_change":"2024-04-26T04:55:08.146580+0000","last_active":"2024-04-26T05:03:25.574926+0000","last_peered":"2024-04-26T05:03:25.574926+0000","last_clean":"2024-04-26T05:03:25.574926+0000","last_became_active":"2024-04-26T04:55:08.146221+0000","last_became_peered":"2024-04-26T04:55:08.146221+0000","last_unstale":"2024-04-26T05:03:25.574926+0000","last_undegraded":"2024-04-26T05:03:25.574926+0000","last_fullsized":"2024-04-26T05:03:25.574926+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:05:30.269062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120224+0000","last_change":"2024-04-26T04:56:48.305114+0000","last_active":"2024-04-26T05:03:23.120224+0000","last_peered":"2024-04-26T05:03:23.120224+0000","last_clean":"2024-04-26T05:03:23.120224+0000","last_became_active":"2024-04-26T04:56:48.304596+0000","last_became_peered":"2024-04-26T04:56:48.304596+0000","last_unstale":"2024-04-26T05:03:23.120224+0000","last_undegraded":"2024-04-26T05:03:23.120224+0000","last_fullsized":"2024-04-26T05:03:23.120224+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:23:08.041144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":924,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782274+0000","last_change":"2024-04-26T04:56:12.719957+0000","last_active":"2024-04-26T05:03:22.782274+0000","last_peered":"2024-04-26T05:03:22.782274+0000","last_clean":"2024-04-26T05:03:22.782274+0000","last_became_active":"2024-04-26T04:56:12.719656+0000","last_became_peered":"2024-04-26T04:56:12.719656+0000","last_unstale":"2024-04-26T05:03:22.782274+0000","last_undegraded":"2024-04-26T05:03:22.782274+0000","last_fullsized":"2024-04-26T05:03:22.782274+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:04:08.753692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":899,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506176+0000","last_change":"2024-04-26T04:45:21.638699+0000","last_active":"2024-04-26T05:03:23.506176+0000","last_peered":"2024-04-26T05:03:23.506176+0000","last_clean":"2024-04-26T05:03:23.506176+0000","last_became_active":"2024-04-26T04:45:21.638390+0000","last_became_peered":"2024-04-26T04:45:21.638390+0000","last_unstale":"2024-04-26T05:03:23.506176+0000","last_undegraded":"2024-04-26T05:03:23.506176+0000","last_fullsized":"2024-04-26T05:03:23.506176+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:29:59.647984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":933,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150412+0000","last_change":"2024-04-26T04:56:48.303190+0000","last_active":"2024-04-26T05:03:23.150412+0000","last_peered":"2024-04-26T05:03:23.150412+0000","last_clean":"2024-04-26T05:03:23.150412+0000","last_became_active":"2024-04-26T04:56:48.302324+0000","last_became_peered":"2024-04-26T04:56:48.302324+0000","last_unstale":"2024-04-26T05:03:23.150412+0000","last_undegraded":"2024-04-26T05:03:23.150412+0000","last_fullsized":"2024-04-26T05:03:23.150412+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:38:28.077865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506153+0000","last_change":"2024-04-26T04:55:08.146674+0000","last_active":"2024-04-26T05:03:23.506153+0000","last_peered":"2024-04-26T05:03:23.506153+0000","last_clean":"2024-04-26T05:03:23.506153+0000","last_became_active":"2024-04-26T04:55:08.146280+0000","last_became_peered":"2024-04-26T04:55:08.146280+0000","last_unstale":"2024-04-26T05:03:23.506153+0000","last_undegraded":"2024-04-26T05:03:23.506153+0000","last_fullsized":"2024-04-26T05:03:23.506153+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:30:30.586691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":899,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120102+0000","last_change":"2024-04-26T04:45:22.642459+0000","last_active":"2024-04-26T05:03:23.120102+0000","last_peered":"2024-04-26T05:03:23.120102+0000","last_clean":"2024-04-26T05:03:23.120102+0000","last_became_active":"2024-04-26T04:45:22.642177+0000","last_became_peered":"2024-04-26T04:45:22.642177+0000","last_unstale":"2024-04-26T05:03:23.120102+0000","last_undegraded":"2024-04-26T05:03:23.120102+0000","last_fullsized":"2024-04-26T05:03:23.120102+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:49:14.494066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506327+0000","last_change":"2024-04-26T04:54:32.846236+0000","last_active":"2024-04-26T05:03:23.506327+0000","last_peered":"2024-04-26T05:03:23.506327+0000","last_clean":"2024-04-26T05:03:23.506327+0000","last_became_active":"2024-04-26T04:54:32.845837+0000","last_became_peered":"2024-04-26T04:54:32.845837+0000","last_unstale":"2024-04-26T05:03:23.506327+0000","last_undegraded":"2024-04-26T05:03:23.506327+0000","last_fullsized":"2024-04-26T05:03:23.506327+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:40:04.649128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1219,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843959+0000","last_change":"2024-04-26T04:56:48.325648+0000","last_active":"2024-04-26T05:03:25.843959+0000","last_peered":"2024-04-26T05:03:25.843959+0000","last_clean":"2024-04-26T05:03:25.843959+0000","last_became_active":"2024-04-26T04:56:48.320226+0000","last_became_peered":"2024-04-26T04:56:48.320226+0000","last_unstale":"2024-04-26T05:03:25.843959+0000","last_undegraded":"2024-04-26T05:03:25.843959+0000","last_fullsized":"2024-04-26T05:03:25.843959+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:39:30.071041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":923,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.781952+0000","last_change":"2024-04-26T04:54:32.835638+0000","last_active":"2024-04-26T05:03:22.781952+0000","last_peered":"2024-04-26T05:03:22.781952+0000","last_clean":"2024-04-26T05:03:22.781952+0000","last_became_active":"2024-04-26T04:54:32.835226+0000","last_became_peered":"2024-04-26T04:54:32.835226+0000","last_unstale":"2024-04-26T05:03:22.781952+0000","last_undegraded":"2024-04-26T05:03:22.781952+0000","last_fullsized":"2024-04-26T05:03:22.781952+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:12:14.882680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574548+0000","last_change":"2024-04-26T04:56:48.306018+0000","last_active":"2024-04-26T05:03:25.574548+0000","last_peered":"2024-04-26T05:03:25.574548+0000","last_clean":"2024-04-26T05:03:25.574548+0000","last_became_active":"2024-04-26T04:56:48.304492+0000","last_became_peered":"2024-04-26T04:56:48.304492+0000","last_unstale":"2024-04-26T05:03:25.574548+0000","last_undegraded":"2024-04-26T05:03:25.574548+0000","last_fullsized":"2024-04-26T05:03:25.574548+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:08:45.182964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":898,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.505712+0000","last_change":"2024-04-26T04:45:21.634430+0000","last_active":"2024-04-26T05:03:23.505712+0000","last_peered":"2024-04-26T05:03:23.505712+0000","last_clean":"2024-04-26T05:03:23.505712+0000","last_became_active":"2024-04-26T04:45:21.634149+0000","last_became_peered":"2024-04-26T04:45:21.634149+0000","last_unstale":"2024-04-26T05:03:23.505712+0000","last_undegraded":"2024-04-26T05:03:23.505712+0000","last_fullsized":"2024-04-26T05:03:23.505712+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:32.326287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120443+0000","last_change":"2024-04-26T04:56:48.349051+0000","last_active":"2024-04-26T05:03:23.120443+0000","last_peered":"2024-04-26T05:03:23.120443+0000","last_clean":"2024-04-26T05:03:23.120443+0000","last_became_active":"2024-04-26T04:56:48.348751+0000","last_became_peered":"2024-04-26T04:56:48.348751+0000","last_unstale":"2024-04-26T05:03:23.120443+0000","last_undegraded":"2024-04-26T05:03:23.120443+0000","last_fullsized":"2024-04-26T05:03:23.120443+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:26:16.122532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574340+0000","last_change":"2024-04-26T04:56:48.351609+0000","last_active":"2024-04-26T05:03:25.574340+0000","last_peered":"2024-04-26T05:03:25.574340+0000","last_clean":"2024-04-26T05:03:25.574340+0000","last_became_active":"2024-04-26T04:56:48.351148+0000","last_became_peered":"2024-04-26T04:56:48.351148+0000","last_unstale":"2024-04-26T05:03:25.574340+0000","last_undegraded":"2024-04-26T05:03:25.574340+0000","last_fullsized":"2024-04-26T05:03:25.574340+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:35:59.658257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1252,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843618+0000","last_change":"2024-04-26T04:56:48.345408+0000","last_active":"2024-04-26T05:03:25.843618+0000","last_peered":"2024-04-26T05:03:25.843618+0000","last_clean":"2024-04-26T05:03:25.843618+0000","last_became_active":"2024-04-26T04:56:48.339993+0000","last_became_peered":"2024-04-26T04:56:48.339993+0000","last_unstale":"2024-04-26T05:03:25.843618+0000","last_undegraded":"2024-04-26T05:03:25.843618+0000","last_fullsized":"2024-04-26T05:03:25.843618+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:13:34.643369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506210+0000","last_change":"2024-04-26T04:56:12.718233+0000","last_active":"2024-04-26T05:03:23.506210+0000","last_peered":"2024-04-26T05:03:23.506210+0000","last_clean":"2024-04-26T05:03:23.506210+0000","last_became_active":"2024-04-26T04:56:12.716681+0000","last_became_peered":"2024-04-26T04:56:12.716681+0000","last_unstale":"2024-04-26T05:03:23.506210+0000","last_undegraded":"2024-04-26T05:03:23.506210+0000","last_fullsized":"2024-04-26T05:03:23.506210+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:53:56.133301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":916,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150768+0000","last_change":"2024-04-26T04:55:08.150455+0000","last_active":"2024-04-26T05:03:23.150768+0000","last_peered":"2024-04-26T05:03:23.150768+0000","last_clean":"2024-04-26T05:03:23.150768+0000","last_became_active":"2024-04-26T04:55:08.149102+0000","last_became_peered":"2024-04-26T04:55:08.149102+0000","last_unstale":"2024-04-26T05:03:23.150768+0000","last_undegraded":"2024-04-26T05:03:23.150768+0000","last_fullsized":"2024-04-26T05:03:23.150768+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:27:55.353586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"346'22","reported_seq":922,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.151048+0000","last_change":"2024-04-26T04:45:21.631617+0000","last_active":"2024-04-26T05:03:23.151048+0000","last_peered":"2024-04-26T05:03:23.151048+0000","last_clean":"2024-04-26T05:03:23.151048+0000","last_became_active":"2024-04-26T04:45:21.631218+0000","last_became_peered":"2024-04-26T04:45:21.631218+0000","last_unstale":"2024-04-26T05:03:23.151048+0000","last_undegraded":"2024-04-26T05:03:23.151048+0000","last_fullsized":"2024-04-26T05:03:23.151048+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:50:12.166307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1298,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842881+0000","last_change":"2024-04-26T04:56:12.756700+0000","last_active":"2024-04-26T05:03:25.842881+0000","last_peered":"2024-04-26T05:03:25.842881+0000","last_clean":"2024-04-26T05:03:25.842881+0000","last_became_active":"2024-04-26T04:56:12.756373+0000","last_became_peered":"2024-04-26T04:56:12.756373+0000","last_unstale":"2024-04-26T05:03:25.842881+0000","last_undegraded":"2024-04-26T05:03:25.842881+0000","last_fullsized":"2024-04-26T05:03:25.842881+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:34:27.746415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":918,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.506051+0000","last_change":"2024-04-26T04:56:48.334715+0000","last_active":"2024-04-26T05:03:23.506051+0000","last_peered":"2024-04-26T05:03:23.506051+0000","last_clean":"2024-04-26T05:03:23.506051+0000","last_became_active":"2024-04-26T04:56:48.333960+0000","last_became_peered":"2024-04-26T04:56:48.333960+0000","last_unstale":"2024-04-26T05:03:23.506051+0000","last_undegraded":"2024-04-26T05:03:23.506051+0000","last_fullsized":"2024-04-26T05:03:23.506051+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:25:42.094675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1208,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.849479+0000","last_change":"2024-04-26T04:56:48.314343+0000","last_active":"2024-04-26T05:03:25.849479+0000","last_peered":"2024-04-26T05:03:25.849479+0000","last_clean":"2024-04-26T05:03:25.849479+0000","last_became_active":"2024-04-26T04:56:48.313758+0000","last_became_peered":"2024-04-26T04:56:48.313758+0000","last_unstale":"2024-04-26T05:03:25.849479+0000","last_undegraded":"2024-04-26T05:03:25.849479+0000","last_fullsized":"2024-04-26T05:03:25.849479+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:53:43.804289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":917,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.120261+0000","last_change":"2024-04-26T04:55:08.152158+0000","last_active":"2024-04-26T05:03:23.120261+0000","last_peered":"2024-04-26T05:03:23.120261+0000","last_clean":"2024-04-26T05:03:23.120261+0000","last_became_active":"2024-04-26T04:55:08.151105+0000","last_became_peered":"2024-04-26T04:55:08.151105+0000","last_unstale":"2024-04-26T05:03:23.120261+0000","last_undegraded":"2024-04-26T05:03:23.120261+0000","last_fullsized":"2024-04-26T05:03:23.120261+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:10:51.526015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1219,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843202+0000","last_change":"2024-04-26T04:56:48.347839+0000","last_active":"2024-04-26T05:03:25.843202+0000","last_peered":"2024-04-26T05:03:25.843202+0000","last_clean":"2024-04-26T05:03:25.843202+0000","last_became_active":"2024-04-26T04:56:48.340171+0000","last_became_peered":"2024-04-26T04:56:48.340171+0000","last_unstale":"2024-04-26T05:03:25.843202+0000","last_undegraded":"2024-04-26T05:03:25.843202+0000","last_fullsized":"2024-04-26T05:03:25.843202+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:07:44.332953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.575472+0000","last_change":"2024-04-26T04:56:48.308107+0000","last_active":"2024-04-26T05:03:25.575472+0000","last_peered":"2024-04-26T05:03:25.575472+0000","last_clean":"2024-04-26T05:03:25.575472+0000","last_became_active":"2024-04-26T04:56:48.307386+0000","last_became_peered":"2024-04-26T04:56:48.307386+0000","last_unstale":"2024-04-26T05:03:25.575472+0000","last_undegraded":"2024-04-26T05:03:25.575472+0000","last_fullsized":"2024-04-26T05:03:25.575472+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:51:00.254568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":914,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:22.782882+0000","last_change":"2024-04-26T04:55:08.149281+0000","last_active":"2024-04-26T05:03:22.782882+0000","last_peered":"2024-04-26T05:03:22.782882+0000","last_clean":"2024-04-26T05:03:22.782882+0000","last_became_active":"2024-04-26T04:55:08.148961+0000","last_became_peered":"2024-04-26T04:55:08.148961+0000","last_unstale":"2024-04-26T05:03:22.782882+0000","last_undegraded":"2024-04-26T05:03:22.782882+0000","last_fullsized":"2024-04-26T05:03:22.782882+0000","mapping_epoch":553,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:12:31.364640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1237,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.843400+0000","last_change":"2024-04-26T04:56:48.342262+0000","last_active":"2024-04-26T05:03:25.843400+0000","last_peered":"2024-04-26T05:03:25.843400+0000","last_clean":"2024-04-26T05:03:25.843400+0000","last_became_active":"2024-04-26T04:56:48.323634+0000","last_became_peered":"2024-04-26T04:56:48.323634+0000","last_unstale":"2024-04-26T05:03:25.843400+0000","last_undegraded":"2024-04-26T05:03:25.843400+0000","last_fullsized":"2024-04-26T05:03:25.843400+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:48:38.243275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1325,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.574841+0000","last_change":"2024-04-26T04:56:48.306483+0000","last_active":"2024-04-26T05:03:25.574841+0000","last_peered":"2024-04-26T05:03:25.574841+0000","last_clean":"2024-04-26T05:03:25.574841+0000","last_became_active":"2024-04-26T04:56:48.304921+0000","last_became_peered":"2024-04-26T04:56:48.304921+0000","last_unstale":"2024-04-26T05:03:25.574841+0000","last_undegraded":"2024-04-26T05:03:25.574841+0000","last_fullsized":"2024-04-26T05:03:25.574841+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:04:58.167596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1297,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.842271+0000","last_change":"2024-04-26T04:56:48.306726+0000","last_active":"2024-04-26T05:03:25.842271+0000","last_peered":"2024-04-26T05:03:25.842271+0000","last_clean":"2024-04-26T05:03:25.842271+0000","last_became_active":"2024-04-26T04:56:48.305823+0000","last_became_peered":"2024-04-26T04:56:48.305823+0000","last_unstale":"2024-04-26T05:03:25.842271+0000","last_undegraded":"2024-04-26T05:03:25.842271+0000","last_fullsized":"2024-04-26T05:03:25.842271+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:17:17.832270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1218,"reported_epoch":963,"state":"active+clean","last_fresh":"2024-04-26T05:03:25.845847+0000","last_change":"2024-04-26T04:56:48.339586+0000","last_active":"2024-04-26T05:03:25.845847+0000","last_peered":"2024-04-26T05:03:25.845847+0000","last_clean":"2024-04-26T05:03:25.845847+0000","last_became_active":"2024-04-26T04:56:48.333394+0000","last_became_peered":"2024-04-26T04:56:48.333394+0000","last_unstale":"2024-04-26T05:03:25.845847+0000","last_undegraded":"2024-04-26T05:03:25.845847+0000","last_fullsized":"2024-04-26T05:03:25.845847+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:21.609530+0000","last_clean_scrub_stamp":"2024-04-26T04:45:21.609530+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:33:54.125128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":935,"reported_epoch":962,"state":"active+clean","last_fresh":"2024-04-26T05:03:23.150946+0000","last_change":"2024-04-26T04:56:12.725815+0000","last_active":"2024-04-26T05:03:23.150946+0000","last_peered":"2024-04-26T05:03:23.150946+0000","last_clean":"2024-04-26T05:03:23.150946+0000","last_became_active":"2024-04-26T04:56:12.725372+0000","last_became_peered":"2024-04-26T04:56:12.725372+0000","last_unstale":"2024-04-26T05:03:23.150946+0000","last_undegraded":"2024-04-26T05:03:23.150946+0000","last_fullsized":"2024-04-26T05:03:23.150946+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:45:20.602484+0000","last_clean_scrub_stamp":"2024-04-26T04:45:20.602484+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:01:20.219547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":954,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201531,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313384,"kb_used_data":16028,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93431768,"statfs":{"total":95995035648,"available":95674130432,"internally_reserved":0,"allocated":16412672,"data_stored":10366066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463168,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325652,"kb_used_data":16008,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93419500,"statfs":{"total":95995035648,"available":95661568000,"internally_reserved":0,"allocated":16392192,"data_stored":10365390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":553,"seq":2375116914799,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65220,"kb_used_data":15416,"kb_used_omap":55,"kb_used_meta":49736,"kb_avail":93679932,"statfs":{"total":95995035648,"available":95928250368,"internally_reserved":0,"allocated":15785984,"data_stored":9770509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":57122,"internal_metadata":50929886},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":519,"seq":2229088026742,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":70100,"kb_used_data":16004,"kb_used_omap":62,"kb_used_meta":54017,"kb_avail":93675052,"statfs":{"total":95995035648,"available":95923253248,"internally_reserved":0,"allocated":16388096,"data_stored":10361469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":63539,"internal_metadata":55314381},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":618,"seq":2654289789026,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65484,"kb_used_data":15424,"kb_used_omap":74,"kb_used_meta":49973,"kb_avail":93679668,"statfs":{"total":95995035648,"available":95927980032,"internally_reserved":0,"allocated":15794176,"data_stored":9771105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76149,"internal_metadata":51173003},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":636,"seq":2731599200347,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":61324,"kb_used_data":15424,"kb_used_omap":65,"kb_used_meta":45822,"kb_avail":93683828,"statfs":{"total":95995035648,"available":95932239872,"internally_reserved":0,"allocated":15794176,"data_stored":9770848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67394,"internal_metadata":46921918},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986440,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316892,"kb_used_data":15440,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93428260,"statfs":{"total":95995035648,"available":95670538240,"internally_reserved":0,"allocated":15810560,"data_stored":9775363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724805,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329164,"kb_used_data":15428,"kb_used_omap":1,"kb_used_meta":313726,"kb_avail":93415988,"statfs":{"total":95995035648,"available":95657971712,"internally_reserved":0,"allocated":15798272,"data_stored":9771378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321255768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":954,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:04:14.464 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T05:04:14.465 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T05:04:14.465 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T05:04:14.465 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:14.742 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:15 smithi111 ceph-mon[86360]: from='client.44975 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:04:15.271 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:15 smithi005 ceph-mon[91801]: from='client.44975 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:04:15.272 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:15 smithi005 ceph-mon[95809]: from='client.44975 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:04:16.211 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:16 smithi005 ceph-mon[91801]: pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:16.212 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:16 smithi005 ceph-mon[95809]: pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:16.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:16 smithi111 ceph-mon[86360]: pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:16.685 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:16.685 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:17.259 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:17.259 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:17 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3611534045' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:17.260 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:17.260 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:17 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3611534045' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:17 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3611534045' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:18 smithi111 ceph-mon[86360]: pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:18.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:18 smithi005 ceph-mon[91801]: pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:18 smithi005 ceph-mon[95809]: pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:20.262 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:20.290 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:20 smithi005 ceph-mon[91801]: pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:20.290 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:20 smithi005 ceph-mon[95809]: pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:20 smithi111 ceph-mon[86360]: pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:20.541 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:22.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:22 smithi005 ceph-mon[91801]: pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:22.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:22 smithi005 ceph-mon[95809]: pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:22.426 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:22.426 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:22 smithi111 ceph-mon[86360]: pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:23 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2164615324' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:23 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2164615324' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:23.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:04:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:04:23] "GET /metrics HTTP/1.1" 200 34238 "" "Prometheus/2.43.0" 2024-04-26T05:04:23.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:23 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2164615324' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:24.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:24 smithi005 ceph-mon[91801]: pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:24.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:24 smithi005 ceph-mon[95809]: pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:24 smithi111 ceph-mon[86360]: pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:25.474 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:25 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:25.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:25 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:25 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:26.054 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:26.301 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:26 smithi111 ceph-mon[86360]: pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:26.356 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:26.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:26 smithi005 ceph-mon[91801]: pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:26 smithi005 ceph-mon[95809]: pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:28.174 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:28 smithi005 ceph-mon[91801]: pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:28.174 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:28 smithi005 ceph-mon[95809]: pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:28 smithi111 ceph-mon[86360]: pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.130 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:29.130 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:29.426 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[91801]: pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3658532565' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[95809]: pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.427 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:29 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3658532565' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:29 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:29 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:29 smithi111 ceph-mon[86360]: pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:29 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3658532565' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:31.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:31.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:31.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:31.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.981 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:31 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:32.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:32.019 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:31 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:04:32.799 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:32.826 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:32.826 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:33.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:33.074 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:33.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:04:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:04:33] "GET /metrics HTTP/1.1" 200 34239 "" "Prometheus/2.43.0" 2024-04-26T05:04:33.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:33 smithi005 ceph-mon[95809]: pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:33.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:33 smithi005 ceph-mon[91801]: pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:34.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:33 smithi111 ceph-mon[86360]: pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:34.905 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:34.905 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:35.228 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:34 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4040743402' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:35.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:34 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4040743402' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:34 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4040743402' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:36.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:35 smithi005 ceph-mon[91801]: pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:36.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:35 smithi005 ceph-mon[95809]: pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:36.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:35 smithi111 ceph-mon[86360]: pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:37.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:37 smithi005 ceph-mon[91801]: pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:37.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:37 smithi005 ceph-mon[95809]: pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:37.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:37 smithi111 ceph-mon[86360]: pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:38.540 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:38.821 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:39.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:39 smithi005 ceph-mon[91801]: pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:39.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:39 smithi005 ceph-mon[95809]: pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:39.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:39 smithi111 ceph-mon[86360]: pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:40.655 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:40.655 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:40.979 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:40 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3532181572' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:40.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:40 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3532181572' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:41.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:40 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3532181572' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:41.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:41 smithi005 ceph-mon[91801]: pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:41.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:41 smithi005 ceph-mon[95809]: pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:42.018 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:41 smithi111 ceph-mon[86360]: pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:43.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:43 smithi005 ceph-mon[91801]: pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:43.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:43 smithi005 ceph-mon[95809]: pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:43.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:04:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:04:43] "GET /metrics HTTP/1.1" 200 34239 "" "Prometheus/2.43.0" 2024-04-26T05:04:43.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:43 smithi111 ceph-mon[86360]: pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:44.265 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:44.542 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:45.601 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:45 smithi005 ceph-mon[91801]: pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:45.602 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:45 smithi005 ceph-mon[95809]: pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:45.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:45 smithi111 ceph-mon[86360]: pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:46.409 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:46.410 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:46.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:46 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1285732363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:46.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:46 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1285732363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:46.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:46 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1285732363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:47.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:47.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:47 smithi005 ceph-mon[91801]: pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:47.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:47.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:47 smithi005 ceph-mon[95809]: pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:47.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:47.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:47 smithi111 ceph-mon[86360]: pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:49.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:49 smithi005 ceph-mon[95809]: pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:49.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:49 smithi005 ceph-mon[91801]: pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:49.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:49 smithi111 ceph-mon[86360]: pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:50.095 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:50.370 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:51.553 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:51 smithi005 ceph-mon[91801]: pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:51.553 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:51 smithi005 ceph-mon[95809]: pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:51.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:51 smithi111 ceph-mon[86360]: pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:52.179 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:52.179 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:52.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:52 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2148694987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:52 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2148694987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:52 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2148694987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:53.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:53 smithi005 ceph-mon[91801]: pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:53.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:53 smithi005 ceph-mon[95809]: pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:53.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:04:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:04:53] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:04:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:53 smithi111 ceph-mon[86360]: pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:55.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:55 smithi005 ceph-mon[91801]: pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:55.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:55 smithi005 ceph-mon[95809]: pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:55.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:55 smithi111 ceph-mon[86360]: pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:55.813 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:56.090 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:57.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:57 smithi005 ceph-mon[91801]: pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:57 smithi005 ceph-mon[95809]: pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:57 smithi111 ceph-mon[86360]: pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:58.010 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:04:58.010 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:58.582 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:58 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3245094629' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:58.583 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:58 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3245094629' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:58.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:58 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3245094629' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:59.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:04:59 smithi005 ceph-mon[91801]: pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:59.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:04:59 smithi005 ceph-mon[95809]: pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:59.770 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:04:59 smithi111 ceph-mon[86360]: pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:01.610 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:01.638 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:01 smithi005 ceph-mon[91801]: pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:01.638 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:01 smithi005 ceph-mon[95809]: pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:01.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:01 smithi111 ceph-mon[86360]: pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:01.886 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:02.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:02.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:02.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:03.292 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:05:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:05:03] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:05:03.773 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:03.774 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:04.088 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:03 smithi005 ceph-mon[95809]: pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:04.088 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:03 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3846070935' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:04.089 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:03 smithi005 ceph-mon[91801]: pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:04.089 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:03 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3846070935' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:03 smithi111 ceph-mon[86360]: pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:04.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:03 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3846070935' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:05.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:05 smithi005 ceph-mon[91801]: pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:05.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:05 smithi005 ceph-mon[95809]: pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:05.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:05 smithi111 ceph-mon[86360]: pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:07.383 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:07.659 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:07.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:07 smithi005 ceph-mon[91801]: pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:07.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:07 smithi005 ceph-mon[95809]: pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:07.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:07 smithi111 ceph-mon[86360]: pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:09.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:09 smithi005 ceph-mon[95809]: pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:09.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:09 smithi005 ceph-mon[91801]: pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:09.595 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:09.596 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:09.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:09 smithi111 ceph-mon[86360]: pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:10.548 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:10 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3649195699' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:10.549 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:10 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3649195699' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:10.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:10 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3649195699' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:11.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:11 smithi005 ceph-mon[91801]: pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:11.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:11 smithi005 ceph-mon[95809]: pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:11.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:11 smithi111 ceph-mon[86360]: pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:13.270 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:13.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:13 smithi005 ceph-mon[91801]: pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:13.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:13 smithi005 ceph-mon[95809]: pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:13.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:05:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:05:13] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:05:13.551 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:13.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:13 smithi111 ceph-mon[86360]: pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:15.351 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:15.352 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:15.352 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:15 smithi005 ceph-mon[91801]: pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:15.352 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:15 smithi005 ceph-mon[95809]: pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:15.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:15 smithi111 ceph-mon[86360]: pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:16.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:16 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2304677755' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:16.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:16 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2304677755' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:16.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:16 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2304677755' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:17.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:17.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:17 smithi005 ceph-mon[91801]: pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:17.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:17.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:17 smithi005 ceph-mon[95809]: pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:17.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:17.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:17 smithi111 ceph-mon[86360]: pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:18.938 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:19.214 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:19.588 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:19 smithi005 ceph-mon[91801]: pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:19.589 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:19 smithi005 ceph-mon[95809]: pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:19.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:19 smithi111 ceph-mon[86360]: pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:21.135 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:21.135 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:21.424 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:21 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2475379234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:21.424 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:21 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2475379234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:21.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:21 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2475379234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:22.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:22 smithi005 ceph-mon[91801]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:22 smithi005 ceph-mon[95809]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:22 smithi111 ceph-mon[86360]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:23.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:23 smithi005 ceph-mon[95809]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:23.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:23 smithi005 ceph-mon[91801]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:23.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:05:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:05:23] "GET /metrics HTTP/1.1" 200 34238 "" "Prometheus/2.43.0" 2024-04-26T05:05:23.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:23 smithi111 ceph-mon[86360]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:24.746 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:25.024 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:25.445 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:25 smithi005 ceph-mon[91801]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:25.445 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:25 smithi005 ceph-mon[95809]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:25.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:25 smithi111 ceph-mon[86360]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:26.848 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:26.848 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:27.148 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:26 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3159430465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:27.148 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:26 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3159430465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:26 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3159430465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:28.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:27 smithi005 ceph-mon[91801]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:28.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:27 smithi005 ceph-mon[95809]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:28.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:27 smithi111 ceph-mon[86360]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:29.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:29 smithi005 ceph-mon[91801]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:29.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:29 smithi005 ceph-mon[95809]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:29.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:29 smithi111 ceph-mon[86360]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:30.452 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:30.729 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:31.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:31 smithi005 ceph-mon[91801]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:31.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:31 smithi005 ceph-mon[95809]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:31.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:31 smithi111 ceph-mon[86360]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:32.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:32.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:32.626 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:32.626 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:33.229 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:05:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:05:33] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2024-04-26T05:05:33.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:33 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2395717249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:33.618 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:33 smithi005 ceph-mon[91801]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:33.618 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:33 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2395717249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:33.619 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:33 smithi005 ceph-mon[95809]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:33.733 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:33 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2395717249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:33.733 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:33 smithi111 ceph-mon[86360]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:36.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[95809]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[91801]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:36 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:36 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:36 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:36 smithi111 ceph-mon[86360]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:36 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:36 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:36.650 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:36.915 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:37.366 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:37 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:37.366 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:37 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:37.366 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:37 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:37.367 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:37 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:37.367 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:37 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:37.367 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:37 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:37 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:37 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:37 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:05:38.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:38 smithi005 ceph-mon[91801]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:38 smithi005 ceph-mon[91801]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:38 smithi005 ceph-mon[95809]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:38 smithi005 ceph-mon[95809]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:38 smithi111 ceph-mon[86360]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:38 smithi111 ceph-mon[86360]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.714 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:38.714 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:39.328 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:39 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2397975800' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:39.328 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:39 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2397975800' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:39 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2397975800' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:40.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:40 smithi005 ceph-mon[91801]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:40.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:40 smithi005 ceph-mon[95809]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:40.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:40 smithi111 ceph-mon[86360]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.330 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:42.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:42 smithi005 ceph-mon[91801]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:42 smithi005 ceph-mon[95809]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:42 smithi111 ceph-mon[86360]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.608 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:05:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:05:43] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2024-04-26T05:05:44.338 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:44 smithi005 ceph-mon[91801]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.339 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:44 smithi005 ceph-mon[95809]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:44 smithi111 ceph-mon[86360]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.608 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:44.608 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:45.192 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:45 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1944236475' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:45.193 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:45 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1944236475' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:45.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:45 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1944236475' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:46.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:46 smithi005 ceph-mon[91801]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:46.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:46 smithi005 ceph-mon[95809]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:46.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:46 smithi111 ceph-mon[86360]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:47.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:47.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:47.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:48.220 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:48.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:48 smithi005 ceph-mon[91801]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:48.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:48 smithi005 ceph-mon[95809]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:48.502 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:48.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:48 smithi111 ceph-mon[86360]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.221 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:50 smithi005 ceph-mon[91801]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.222 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:50 smithi005 ceph-mon[95809]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.461 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:50.462 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:50.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:50 smithi111 ceph-mon[86360]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:51.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:51 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3815544227' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:51.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:51 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3815544227' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:51.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:51 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3815544227' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:52.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:52 smithi005 ceph-mon[91801]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:52.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:52 smithi005 ceph-mon[95809]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:52.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:52 smithi111 ceph-mon[86360]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:05:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:05:53] "GET /metrics HTTP/1.1" 200 34237 "" "Prometheus/2.43.0" 2024-04-26T05:05:54.077 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:54.356 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:54.475 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:54 smithi005 ceph-mon[91801]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:54.476 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:54 smithi005 ceph-mon[95809]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:54.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:54 smithi111 ceph-mon[86360]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.200 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:05:56.200 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:56.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:56 smithi005 ceph-mon[91801]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:56 smithi005 ceph-mon[95809]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:56 smithi111 ceph-mon[86360]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:57.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:57 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1507772556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:57.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:57 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1507772556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:57.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:57 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1507772556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:58.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:05:58 smithi005 ceph-mon[91801]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:58.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:05:58 smithi005 ceph-mon[95809]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:58.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:05:58 smithi111 ceph-mon[86360]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:59.826 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:00.103 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:00.451 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:00 smithi005 ceph-mon[95809]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.452 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:00 smithi005 ceph-mon[91801]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:00 smithi111 ceph-mon[86360]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:02.075 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:02.075 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:02.395 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:02 smithi005 ceph-mon[91801]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:02.396 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:02.396 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:02 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/246316314' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:02.396 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:02 smithi005 ceph-mon[95809]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:02.396 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:02.396 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:02 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/246316314' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:02 smithi111 ceph-mon[86360]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:02 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/246316314' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:03.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:06:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:06:03] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:06:04.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:04 smithi005 ceph-mon[91801]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:04.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:04 smithi005 ceph-mon[95809]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:04.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:04 smithi111 ceph-mon[86360]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:05.744 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:06.022 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:06.417 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:06 smithi005 ceph-mon[91801]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:06.418 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:06 smithi005 ceph-mon[95809]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:06 smithi111 ceph-mon[86360]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:07.892 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:07.892 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:08 smithi005 ceph-mon[91801]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:08.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:08 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/726881503' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:08 smithi005 ceph-mon[95809]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:08 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/726881503' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:08 smithi111 ceph-mon[86360]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:08.519 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:08 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/726881503' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:10.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:10 smithi005 ceph-mon[95809]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:10.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:10 smithi005 ceph-mon[91801]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:10.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:10 smithi111 ceph-mon[86360]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:11.538 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:11.816 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:12.207 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:12 smithi005 ceph-mon[91801]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:12.207 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:12 smithi005 ceph-mon[95809]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:12 smithi111 ceph-mon[86360]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:06:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:06:13] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:06:13.568 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:13.568 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:14.450 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:14 smithi005 ceph-mon[91801]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:14.451 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:14 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3785996435' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:14.451 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:14 smithi005 ceph-mon[95809]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:14.451 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:14 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3785996435' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:14 smithi111 ceph-mon[86360]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:14 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3785996435' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:16.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:16 smithi005 ceph-mon[91801]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:16 smithi005 ceph-mon[95809]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:16 smithi111 ceph-mon[86360]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:17.181 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:17.460 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:17.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:17.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:17.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:18.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:18 smithi005 ceph-mon[91801]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:18.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:18 smithi005 ceph-mon[95809]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:18 smithi111 ceph-mon[86360]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:19.225 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:19.225 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:20.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:20 smithi005 ceph-mon[91801]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:20 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2124979875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:20 smithi005 ceph-mon[95809]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:20 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2124979875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:20 smithi111 ceph-mon[86360]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:20 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2124979875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:22.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:22 smithi005 ceph-mon[91801]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:22 smithi005 ceph-mon[95809]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:22 smithi111 ceph-mon[86360]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.837 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:23.113 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:23.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:06:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:06:23] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:06:24.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:24 smithi005 ceph-mon[95809]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:24.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:24 smithi005 ceph-mon[91801]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:24.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:24 smithi111 ceph-mon[86360]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:25.015 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:25.015 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:25.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:25 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/776319772' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:25.381 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:25 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/776319772' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:25.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:25 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/776319772' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:26.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:26 smithi005 ceph-mon[91801]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:26 smithi005 ceph-mon[95809]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:26 smithi111 ceph-mon[86360]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:28 smithi005 ceph-mon[91801]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:28 smithi005 ceph-mon[95809]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:28 smithi111 ceph-mon[86360]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.822 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:29.100 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:30.382 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:30 smithi005 ceph-mon[91801]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:30.383 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:30 smithi005 ceph-mon[95809]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:30.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:30 smithi111 ceph-mon[86360]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:30.888 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:30.888 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:31.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:31 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2383213955' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:31.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:31 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2383213955' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:31.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:31 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2383213955' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:32.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:32 smithi005 ceph-mon[91801]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:32.483 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:32.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:32 smithi005 ceph-mon[95809]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:32.483 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:32 smithi111 ceph-mon[86360]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:33.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:06:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:06:33] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2024-04-26T05:06:34.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:34 smithi005 ceph-mon[91801]: pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:34.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:34 smithi005 ceph-mon[95809]: pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:34.508 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:34.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:34 smithi111 ceph-mon[86360]: pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:34.786 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:36.450 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:36 smithi005 ceph-mon[91801]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:36.451 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:36 smithi005 ceph-mon[95809]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:36.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:36 smithi111 ceph-mon[86360]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:36.709 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:36.709 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:37.293 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:37 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:37.293 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:37 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2517120258' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:37.294 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:37 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:37.294 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:37 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2517120258' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:37 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:06:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:37 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2517120258' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:38.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:38 smithi111 ceph-mon[86360]: pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:38.560 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:38 smithi005 ceph-mon[91801]: pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:38.561 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:38 smithi005 ceph-mon[95809]: pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:40.295 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:40.517 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:40 smithi111 ceph-mon[86360]: pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:40.593 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:40.722 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:40 smithi005 ceph-mon[91801]: pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:40.722 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:40 smithi005 ceph-mon[95809]: pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[95809]: pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.661 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.662 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.662 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:41 smithi005 ceph-mon[91801]: pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:41 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:41 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:41 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:41 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:41.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:41 smithi111 ceph-mon[86360]: pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.419 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:42.419 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[95809]: pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[95809]: pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[91801]: pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[91801]: pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.727 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:42 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:42 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:42 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:42 smithi111 ceph-mon[86360]: pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:42 smithi111 ceph-mon[86360]: pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:42 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:06:43.421 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:06:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:06:43] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2024-04-26T05:06:43.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:43 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3294476283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:43.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:43 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3294476283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:43.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:43 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3294476283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:44.708 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:44 smithi005 ceph-mon[91801]: pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:44.708 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:44 smithi005 ceph-mon[95809]: pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:44.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:44 smithi111 ceph-mon[86360]: pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:46.054 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:46.323 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:47.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:46 smithi005 ceph-mon[91801]: pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.159 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:47.159 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:46 smithi005 ceph-mon[95809]: pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.159 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:46 smithi111 ceph-mon[86360]: pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:48.267 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:48.267 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:48 smithi005 ceph-mon[91801]: pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:48 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2643993313' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:48 smithi005 ceph-mon[95809]: pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:48 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2643993313' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:48 smithi111 ceph-mon[86360]: pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:48 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2643993313' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:50 smithi005 ceph-mon[91801]: pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:50 smithi005 ceph-mon[95809]: pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:50 smithi111 ceph-mon[86360]: pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:51.928 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:52.207 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:53.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:52 smithi005 ceph-mon[91801]: pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:52 smithi005 ceph-mon[95809]: pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:06:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:06:53] "GET /metrics HTTP/1.1" 200 34239 "" "Prometheus/2.43.0" 2024-04-26T05:06:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:52 smithi111 ceph-mon[86360]: pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:54.118 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:54.119 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:54 smithi005 ceph-mon[91801]: pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:54 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2696616209' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:54 smithi005 ceph-mon[95809]: pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:54 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2696616209' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:54 smithi111 ceph-mon[86360]: pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:54 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2696616209' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:57.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:56 smithi005 ceph-mon[91801]: pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:56 smithi005 ceph-mon[95809]: pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:56 smithi111 ceph-mon[86360]: pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.789 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:58.068 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:59.124 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:58 smithi005 ceph-mon[91801]: pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.125 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:58 smithi005 ceph-mon[95809]: pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:58 smithi111 ceph-mon[86360]: pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.911 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:06:59.912 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:00.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:06:59 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1539231455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:00.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:06:59 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1539231455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:06:59 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1539231455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:01.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:00 smithi005 ceph-mon[91801]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:00 smithi005 ceph-mon[95809]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:00 smithi111 ceph-mon[86360]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:02.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:03.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:02 smithi005 ceph-mon[91801]: pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:02 smithi005 ceph-mon[95809]: pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:07:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:07:03] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-04-26T05:07:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:02 smithi111 ceph-mon[86360]: pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.551 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:03.830 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:05.186 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:04 smithi005 ceph-mon[95809]: pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.186 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:04 smithi005 ceph-mon[91801]: pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:04 smithi111 ceph-mon[86360]: pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.706 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:05.707 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:06.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:05 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1099569436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:06.296 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:05 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1099569436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:06.296 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:05 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1099569436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:07.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:06 smithi111 ceph-mon[86360]: pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:07.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:06 smithi005 ceph-mon[91801]: pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:07.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:06 smithi005 ceph-mon[95809]: pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:08 smithi111 ceph-mon[86360]: pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:09.325 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:09.352 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:08 smithi005 ceph-mon[91801]: pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:09.353 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:08 smithi005 ceph-mon[95809]: pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:09.600 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:11.249 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:10 smithi005 ceph-mon[95809]: pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.249 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:10 smithi005 ceph-mon[91801]: pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:10 smithi111 ceph-mon[86360]: pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.537 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:11.537 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:12.178 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:11 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/679040411' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:12.178 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:11 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/679040411' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:12.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:11 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/679040411' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:13.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:12 smithi111 ceph-mon[86360]: pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:13.381 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:12 smithi005 ceph-mon[91801]: pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:13.382 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:12 smithi005 ceph-mon[95809]: pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:13.382 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:07:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:07:13] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-04-26T05:07:15.209 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:14 smithi111 ceph-mon[86360]: pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:15.479 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:15.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:14 smithi005 ceph-mon[95809]: pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:15.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:14 smithi005 ceph-mon[91801]: pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:16 smithi005 ceph-mon[91801]: pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:16 smithi005 ceph-mon[95809]: pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:16 smithi111 ceph-mon[86360]: pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:17.410 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:17.410 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:18.102 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:18 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/480560807' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:18.102 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:18 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/480560807' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:18.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:18 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/480560807' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:19 smithi111 ceph-mon[86360]: pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:19.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:19 smithi005 ceph-mon[91801]: pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:19.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:19 smithi005 ceph-mon[95809]: pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:21.132 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:21.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:21 smithi111 ceph-mon[86360]: pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:21.418 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:21.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:21 smithi005 ceph-mon[91801]: pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:21.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:21 smithi005 ceph-mon[95809]: pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.081 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:23 smithi005 ceph-mon[91801]: pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.082 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:23 smithi005 ceph-mon[95809]: pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:23 smithi111 ceph-mon[86360]: pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.454 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:23.455 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:23.455 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:07:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:07:23] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-04-26T05:07:24.042 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:24 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2143320798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:24.043 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:24 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2143320798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:24.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:24 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2143320798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:25 smithi111 ceph-mon[86360]: pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:25.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:25 smithi005 ceph-mon[91801]: pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:25.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:25 smithi005 ceph-mon[95809]: pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:27.069 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:27.348 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:27.471 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:27 smithi005 ceph-mon[91801]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:27.471 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:27 smithi005 ceph-mon[95809]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:27.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:27 smithi111 ceph-mon[86360]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:29 smithi005 ceph-mon[91801]: pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:29 smithi005 ceph-mon[95809]: pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:29.330 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:29.330 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:29.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:29 smithi111 ceph-mon[86360]: pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:30.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:30 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/593060013' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:30.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:30 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/593060013' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:30.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:30 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/593060013' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:31.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:31 smithi005 ceph-mon[91801]: pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:31.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:31 smithi005 ceph-mon[95809]: pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:31.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:31 smithi111 ceph-mon[86360]: pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:32.482 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:32 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:32.482 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:32 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:32.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:32 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:32.981 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:33.280 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:33.403 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:33 smithi005 ceph-mon[91801]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:33.403 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:33 smithi005 ceph-mon[95809]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:33.404 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:07:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:07:33] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:07:33.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:33 smithi111 ceph-mon[86360]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:35.134 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:35.135 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:35.419 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:35 smithi005 ceph-mon[91801]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:35.420 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:35 smithi005 ceph-mon[95809]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:35.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:35 smithi111 ceph-mon[86360]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:36.419 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:36 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/4253028423' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:36.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:36 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/4253028423' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:36.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:36 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/4253028423' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:37.479 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:37 smithi005 ceph-mon[95809]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:37.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:37 smithi005 ceph-mon[91801]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:37.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:37 smithi111 ceph-mon[86360]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:38.715 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:38.992 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:39.392 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:39 smithi005 ceph-mon[95809]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:39.392 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:39 smithi005 ceph-mon[91801]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:39.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:39 smithi111 ceph-mon[86360]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:40.918 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:40.918 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:41 smithi005 ceph-mon[91801]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:41.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:41 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3966420640' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:41 smithi005 ceph-mon[95809]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:41.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:41 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3966420640' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:41.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:41 smithi111 ceph-mon[86360]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:41.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:41 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3966420640' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:42.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:42 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:42.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:42 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:42.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:42 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:43.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:43 smithi005 ceph-mon[95809]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:43.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:43 smithi005 ceph-mon[91801]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:43 smithi111 ceph-mon[86360]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:43.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:07:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:07:43] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:07:44.531 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:45.158 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:45.366 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:45 smithi111 ceph-mon[86360]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:45.385 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:45 smithi005 ceph-mon[91801]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:45.386 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:45 smithi005 ceph-mon[95809]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:47.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[91801]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:47.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:47.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.155 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.156 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.156 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.156 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[95809]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:47.156 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:47.156 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.156 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:46 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:46 smithi111 ceph-mon[86360]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:46 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:47.533 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:47.533 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:48.109 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:48.109 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:48.109 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:48.109 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3855407480' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:48.109 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:48.110 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:48.110 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:48.110 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:47 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3855407480' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:47 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:07:48.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:47 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3855407480' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:48 smithi005 ceph-mon[91801]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:48 smithi005 ceph-mon[91801]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:48 smithi005 ceph-mon[95809]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:48 smithi005 ceph-mon[95809]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:48 smithi111 ceph-mon[86360]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:48 smithi111 ceph-mon[86360]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.139 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:51.167 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:50 smithi005 ceph-mon[91801]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.167 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:50 smithi005 ceph-mon[95809]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:50 smithi111 ceph-mon[86360]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.417 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:53.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:52 smithi005 ceph-mon[91801]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:53.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:52 smithi005 ceph-mon[95809]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:53.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:52 smithi111 ceph-mon[86360]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:53.346 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:53.346 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:53.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:07:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:07:53] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-04-26T05:07:53.936 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:53 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1858251168' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:53.937 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:53 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1858251168' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:54.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:53 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1858251168' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:55.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:54 smithi005 ceph-mon[91801]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:55.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:54 smithi005 ceph-mon[95809]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:55.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:54 smithi111 ceph-mon[86360]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.939 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:57.215 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:57.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:56 smithi005 ceph-mon[91801]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:57.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:56 smithi005 ceph-mon[95809]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:57.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:56 smithi111 ceph-mon[86360]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.980 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:58 smithi005 ceph-mon[91801]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:58 smithi005 ceph-mon[95809]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:59.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:58 smithi111 ceph-mon[86360]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:59.361 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:07:59.361 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:59.949 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:07:59 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1370710865' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:59.950 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:07:59 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1370710865' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:00.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:07:59 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1370710865' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:01.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:00 smithi005 ceph-mon[91801]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:01.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:00 smithi005 ceph-mon[95809]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:01.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:00 smithi111 ceph-mon[86360]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:02.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:01 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:02.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:01 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:02.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:01 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:02.976 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:03.195 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:02 smithi005 ceph-mon[91801]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:03.195 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:02 smithi005 ceph-mon[95809]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:03.255 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:03.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:02 smithi111 ceph-mon[86360]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:03.448 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:08:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:08:03] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:08:05.107 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:05.107 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:05.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:04 smithi005 ceph-mon[91801]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:05.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:04 smithi005 ceph-mon[95809]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:05.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:04 smithi111 ceph-mon[86360]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:05.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:05 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1807212734' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:05.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:05 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1807212734' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:06.169 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:05 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1807212734' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:07.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:06 smithi005 ceph-mon[91801]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:07.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:06 smithi005 ceph-mon[95809]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:07.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:06 smithi111 ceph-mon[86360]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:08.700 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:08.979 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:08 smithi005 ceph-mon[91801]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:08.980 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:08 smithi005 ceph-mon[95809]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:08.980 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:09.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:08 smithi111 ceph-mon[86360]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:10.811 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:10.811 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:11.208 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:10 smithi005 ceph-mon[91801]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.208 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:10 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/2823582047' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:11.208 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:10 smithi005 ceph-mon[95809]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.208 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:10 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/2823582047' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:10 smithi111 ceph-mon[86360]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:10 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/2823582047' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:13.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:12 smithi005 ceph-mon[91801]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:13.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:12 smithi005 ceph-mon[95809]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:13.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:12 smithi111 ceph-mon[86360]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:13.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:08:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:08:13] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-04-26T05:08:14.431 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:14.698 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:15.165 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:14 smithi005 ceph-mon[91801]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:15.165 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:14 smithi005 ceph-mon[95809]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:15.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:14 smithi111 ceph-mon[86360]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:16.624 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:16.625 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:17.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:16 smithi005 ceph-mon[91801]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:16 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:17.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:16 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1507928943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:16 smithi005 ceph-mon[95809]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:16 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:17.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:16 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1507928943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:16 smithi111 ceph-mon[86360]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:16 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:17.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:16 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1507928943' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:19.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:18 smithi005 ceph-mon[91801]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:19.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:18 smithi005 ceph-mon[95809]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:19.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:18 smithi111 ceph-mon[86360]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:20.261 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:20.539 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:21.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:20 smithi005 ceph-mon[91801]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:21.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:20 smithi005 ceph-mon[95809]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:21.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:20 smithi111 ceph-mon[86360]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:22.646 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:22.646 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:23.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:22 smithi005 ceph-mon[91801]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:23.196 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:22 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3711913573' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:22 smithi005 ceph-mon[95809]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:23.196 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:22 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3711913573' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:22 smithi111 ceph-mon[86360]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:23.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:22 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3711913573' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:23.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:08:23 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:08:23] "GET /metrics HTTP/1.1" 200 34238 "" "Prometheus/2.43.0" 2024-04-26T05:08:25.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:24 smithi005 ceph-mon[91801]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:24 smithi005 ceph-mon[95809]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:24 smithi111 ceph-mon[86360]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:26.286 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:26.563 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:27.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:26 smithi005 ceph-mon[91801]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:27.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:26 smithi005 ceph-mon[95809]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:27.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:26 smithi111 ceph-mon[86360]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:28.630 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:28.630 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:28 smithi005 ceph-mon[91801]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:29.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:28 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3394492462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:28 smithi005 ceph-mon[95809]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:29.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:28 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3394492462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:29.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:28 smithi111 ceph-mon[86360]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:29.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:28 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3394492462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:31.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:30 smithi005 ceph-mon[91801]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:30 smithi005 ceph-mon[95809]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:30 smithi111 ceph-mon[86360]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:32.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:31 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:32.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:31 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:32.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:31 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:32.325 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:32.606 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:33.061 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:32 smithi005 ceph-mon[95809]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.061 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:32 smithi005 ceph-mon[91801]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:32 smithi111 ceph-mon[86360]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:08:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:08:33] "GET /metrics HTTP/1.1" 200 34237 "" "Prometheus/2.43.0" 2024-04-26T05:08:34.612 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:34.612 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:35.197 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:34 smithi005 ceph-mon[91801]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:35.198 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:34 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1717457444' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:35.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:34 smithi005 ceph-mon[95809]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:35.198 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:34 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1717457444' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:34 smithi111 ceph-mon[86360]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:35.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:34 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1717457444' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:37.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:36 smithi005 ceph-mon[91801]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:36 smithi005 ceph-mon[95809]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:36 smithi111 ceph-mon[86360]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:38.228 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:38.507 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:39.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:38 smithi005 ceph-mon[91801]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:39.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:38 smithi005 ceph-mon[95809]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:39.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:38 smithi111 ceph-mon[86360]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:40.455 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:40.456 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:41.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:40 smithi005 ceph-mon[95809]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:41.112 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:40 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3739011947' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:41.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:40 smithi005 ceph-mon[91801]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:41.113 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:40 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3739011947' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:40 smithi111 ceph-mon[86360]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:41.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:40 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3739011947' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:43.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:42 smithi005 ceph-mon[91801]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:43.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:42 smithi005 ceph-mon[95809]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:43.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:08:43 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:08:43] "GET /metrics HTTP/1.1" 200 34237 "" "Prometheus/2.43.0" 2024-04-26T05:08:43.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:42 smithi111 ceph-mon[86360]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:44.143 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:44.410 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:45.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:44 smithi005 ceph-mon[95809]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:45.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:44 smithi005 ceph-mon[91801]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:45.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:44 smithi111 ceph-mon[86360]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:46.466 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:46.466 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:47.010 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:46 smithi005 ceph-mon[91801]: pgmap v1304: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:47.011 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:46 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3978434638' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:47.011 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:46 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:47.011 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:46 smithi005 ceph-mon[95809]: pgmap v1304: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:47.011 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:46 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3978434638' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:47.011 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:46 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:46 smithi111 ceph-mon[86360]: pgmap v1304: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:46 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3978434638' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:47.268 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:46 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:48.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:47 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:48.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:47 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:48.270 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:47 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:49.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:48 smithi005 ceph-mon[91801]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:49.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:48 smithi005 ceph-mon[95809]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:49.231 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:48 smithi111 ceph-mon[86360]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:50.013 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:50.596 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:51.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:50 smithi005 ceph-mon[91801]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:51.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:50 smithi005 ceph-mon[95809]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:51.267 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:50 smithi111 ceph-mon[86360]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:52.696 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:52 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:52.696 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:52 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:52.696 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:52 smithi005 ceph-mon[91801]: pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:52.696 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:52 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:52.696 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:52 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:52.696 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:52 smithi005 ceph-mon[95809]: pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:52 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:52 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:52.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:52 smithi111 ceph-mon[86360]: pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.068 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:53.068 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:53.333 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:08:53] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2024-04-26T05:08:53.729 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/1001313274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:53.730 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:53.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.731 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:53 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/1001313274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: from='mgr.34107 ' entity='mgr.y' 2024-04-26T05:08:53.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:53 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/1001313274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:56.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:56 smithi005 ceph-mon[91801]: pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:56.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:56 smithi005 ceph-mon[95809]: pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:56.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:56 smithi111 ceph-mon[86360]: pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:56.798 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:57.077 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:58.399 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:08:58 smithi005 ceph-mon[91801]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:58.400 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:08:58 smithi005 ceph-mon[95809]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:58.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:08:58 smithi111 ceph-mon[86360]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:59.185 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:08:59.185 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:00.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:00 smithi005 ceph-mon[95809]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:00.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:00 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3114899585' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:00.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:00 smithi005 ceph-mon[91801]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:00.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:00 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3114899585' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:00.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:00 smithi111 ceph-mon[86360]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:00.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:00 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3114899585' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:02.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:02 smithi005 ceph-mon[91801]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:02.480 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:02 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:02 smithi005 ceph-mon[95809]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:02.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:02 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:02 smithi111 ceph-mon[86360]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:02.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:02 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:02.843 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:03.123 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:03.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:03 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:09:03] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-04-26T05:09:04.420 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:04 smithi005 ceph-mon[91801]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:04.420 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:04 smithi005 ceph-mon[95809]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:04.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:04 smithi111 ceph-mon[86360]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:05.210 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:09:05.210 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:06.229 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:06 smithi005 ceph-mon[91801]: pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:06.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:06 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/553956737' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:06 smithi005 ceph-mon[95809]: pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:06.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:06 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/553956737' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:06 smithi111 ceph-mon[86360]: pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:06.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:06 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/553956737' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:08.320 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:08 smithi111 ceph-mon[86360]: pgmap v1316: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:08.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:08 smithi005 ceph-mon[91801]: pgmap v1316: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:08.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:08 smithi005 ceph-mon[95809]: pgmap v1316: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:08.934 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:09.213 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:10.363 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:10 smithi005 ceph-mon[91801]: pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:10.364 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:10 smithi005 ceph-mon[95809]: pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:10.507 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:10 smithi111 ceph-mon[86360]: pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:11.188 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:09:11.188 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:12 smithi005 ceph-mon[91801]: pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:12 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/25801707' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:12 smithi005 ceph-mon[95809]: pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:12 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/25801707' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:12 smithi111 ceph-mon[86360]: pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:12 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/25801707' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:13.479 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:13 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: ::ffff:172.21.15.111 - - [26/Apr/2024:05:09:13] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-04-26T05:09:14.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:14 smithi005 ceph-mon[91801]: pgmap v1319: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:14 smithi005 ceph-mon[95809]: pgmap v1319: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:14 smithi111 ceph-mon[86360]: pgmap v1319: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.844 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:15.134 INFO:teuthology.orchestra.run.smithi005.stderr:Inferring config /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:16.421 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:16 smithi005 ceph-mon[91801]: pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.421 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:16 smithi005 ceph-mon[95809]: pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:16 smithi111 ceph-mon[86360]: pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:17.204 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-26T05:09:17.204 INFO:teuthology.orchestra.run.smithi005.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:17.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:17 smithi005 ceph-mon[91801]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:17.480 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:17 smithi005 ceph-mon[95809]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:17.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:17 smithi111 ceph-mon[86360]: from='mgr.34107 172.21.15.5:0/1581105148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:17.846 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:09:18.084 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e7761ddcfe7b4502affae7ba5bf6666c Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:09:18.086 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T05:09:18.095 INFO:tasks.cephadm:Teardown begin 2024-04-26T05:09:18.096 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:09:18.124 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:18 smithi005 ceph-mon[91801]: pgmap v1321: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:18.125 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:18 smithi005 ceph-mon[91801]: from='client.? 172.21.15.5:0/3396605483' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:18.125 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:18 smithi005 ceph-mon[95809]: pgmap v1321: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:18.125 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:18 smithi005 ceph-mon[95809]: from='client.? 172.21.15.5:0/3396605483' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:18.130 DEBUG:teuthology.orchestra.run.smithi111:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:09:18.162 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T05:09:18.162 DEBUG:teuthology.orchestra.run.smithi005:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:09:18.186 DEBUG:teuthology.orchestra.run.smithi111:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:09:18.219 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T05:09:18.219 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-26T05:09:18.219 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a 2024-04-26T05:09:18.479 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:18 smithi005 systemd[1]: Stopping Ceph mon.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:18 smithi111 ceph-mon[86360]: pgmap v1321: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:18.518 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:18 smithi111 ceph-mon[86360]: from='client.? 172.21.15.5:0/3396605483' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:18.958 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:18 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[91797]: 2024-04-26T05:09:18.539+0000 7f8a6fbdc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:18.958 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:18 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a[91797]: 2024-04-26T05:09:18.539+0000 7f8a6fbdc700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-26T05:09:18.959 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:18 smithi005 podman[144754]: 2024-04-26 05:09:18.683145305 +0000 UTC m=+0.258830309 container died 249028d8f146fb810ee0178b558c8e248685131d22be61c5638cda751b1290ad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:09:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:19 smithi005 podman[144754]: 2024-04-26 05:09:19.056781436 +0000 UTC m=+0.632466437 container cleanup 249028d8f146fb810ee0178b558c8e248685131d22be61c5638cda751b1290ad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5) 2024-04-26T05:09:19.230 INFO:journalctl@ceph.mon.a.smithi005.stdout:Apr 26 05:09:19 smithi005 bash[144754]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-a 2024-04-26T05:09:19.491 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.a.service' 2024-04-26T05:09:19.538 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:19.539 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-26T05:09:19.539 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-26T05:09:19.539 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.c 2024-04-26T05:09:19.853 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:19 smithi005 systemd[1]: Stopping Ceph mon.c for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:20.229 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c[95805]: 2024-04-26T05:09:19.852+0000 7f1649e38700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c[95805]: 2024-04-26T05:09:19.852+0000 7f1649e38700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:09:20.230 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:20 smithi005 podman[144876]: 2024-04-26 05:09:20.016104643 +0000 UTC m=+0.288853525 container died 9834a90fbbaaa3e3e9055264d6cad05970748731c137afda54ef380c7231964f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:09:20.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:05:09:19] ENGINE Bus STOPPING 2024-04-26T05:09:20.230 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:05:09:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T05:09:20.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:05:09:19] ENGINE Bus STOPPED 2024-04-26T05:09:20.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:05:09:19] ENGINE Bus STARTING 2024-04-26T05:09:20.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:05:09:19] ENGINE Serving on http://:::9283 2024-04-26T05:09:20.231 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:19 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: [26/Apr/2024:05:09:19] ENGINE Bus STARTED 2024-04-26T05:09:20.640 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:20 smithi005 podman[144876]: 2024-04-26 05:09:20.378404256 +0000 UTC m=+0.651153145 container cleanup 9834a90fbbaaa3e3e9055264d6cad05970748731c137afda54ef380c7231964f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-04-26T05:09:20.641 INFO:journalctl@ceph.mon.c.smithi005.stdout:Apr 26 05:09:20 smithi005 bash[144876]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-c 2024-04-26T05:09:20.812 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.c.service' 2024-04-26T05:09:20.845 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:20.846 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-26T05:09:20.846 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-26T05:09:20.846 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.b 2024-04-26T05:09:21.147 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:20 smithi111 systemd[1]: Stopping Ceph mon.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:21.490 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b[86356]: 2024-04-26T05:09:21.145+0000 7f3429872700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:21.490 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b[86356]: 2024-04-26T05:09:21.145+0000 7f3429872700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:09:21.490 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:21 smithi111 podman[99107]: 2024-04-26 05:09:21.198314906 +0000 UTC m=+0.160114450 container died c986c5c74b8097a4ccf42367789d100f01ad42fa8da95fadad14a42ed679ad76 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:09:21.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:21 smithi111 podman[99107]: 2024-04-26 05:09:21.679976134 +0000 UTC m=+0.641775736 container cleanup c986c5c74b8097a4ccf42367789d100f01ad42fa8da95fadad14a42ed679ad76 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb) 2024-04-26T05:09:21.768 INFO:journalctl@ceph.mon.b.smithi111.stdout:Apr 26 05:09:21 smithi111 bash[99107]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mon-b 2024-04-26T05:09:22.144 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mon.b.service' 2024-04-26T05:09:22.191 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:22.191 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-26T05:09:22.191 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-26T05:09:22.191 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y 2024-04-26T05:09:22.480 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:22 smithi005 systemd[1]: Stopping Ceph mgr.y for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:22.778 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T05:09:22.495+0000 7f1615376700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:22.778 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:22 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y[83190]: 2024-04-26T05:09:22.495+0000 7f1615376700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-26T05:09:22.778 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:22 smithi005 podman[145008]: 2024-04-26 05:09:22.527349271 +0000 UTC m=+0.140384656 container died c895550b1a50d0276896cedbe4f9108819594711d6b60a0372e8cf1d715c8f77 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T05:09:23.103 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:22 smithi005 podman[145008]: 2024-04-26 05:09:22.943836917 +0000 UTC m=+0.556872349 container cleanup c895550b1a50d0276896cedbe4f9108819594711d6b60a0372e8cf1d715c8f77 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-26T05:09:23.104 INFO:journalctl@ceph.mgr.y.smithi005.stdout:Apr 26 05:09:22 smithi005 bash[145008]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-y 2024-04-26T05:09:23.328 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-26T05:09:23.364 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:23.365 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-26T05:09:23.365 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-26T05:09:23.365 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x 2024-04-26T05:09:23.707 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 05:09:23 smithi111 systemd[1]: Stopping Ceph mgr.x for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:23.969 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 05:09:23 smithi111 podman[99228]: 2024-04-26 05:09:23.707200187 +0000 UTC m=+0.150850871 container died a0572da61dd7926675f750205df704329e896cf882fd56bd7630e24c54c9ff3e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-26T05:09:24.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 05:09:24 smithi111 podman[99228]: 2024-04-26 05:09:24.034826991 +0000 UTC m=+0.478477674 container cleanup a0572da61dd7926675f750205df704329e896cf882fd56bd7630e24c54c9ff3e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:09:24.268 INFO:journalctl@ceph.mgr.x.smithi111.stdout:Apr 26 05:09:24 smithi111 bash[99228]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-mgr-x 2024-04-26T05:09:24.596 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-26T05:09:24.628 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:24.628 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-26T05:09:24.628 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T05:09:24.628 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.0 2024-04-26T05:09:24.654 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:24.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:24.655 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:24.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:24.923 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:24 smithi005 systemd[1]: Stopping Ceph osd.0 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:25.229 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[104799]: 2024-04-26T05:09:24.922+0000 7ff393124700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:25.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[104799]: 2024-04-26T05:09:24.922+0000 7ff393124700 -1 osd.0 963 *** Got signal Terminated *** 2024-04-26T05:09:25.230 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0[104799]: 2024-04-26T05:09:24.922+0000 7ff393124700 -1 osd.0 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:09:30.463 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:30.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:30.464 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:30.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:30.724 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:30 smithi005 podman[145129]: 2024-04-26 05:09:30.463250076 +0000 UTC m=+5.637476349 container died d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T05:09:30.979 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:30 smithi005 podman[145129]: 2024-04-26 05:09:30.923171525 +0000 UTC m=+6.097397805 container cleanup d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T05:09:30.980 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:30 smithi005 bash[145129]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0 2024-04-26T05:09:31.479 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:31 smithi005 podman[145145]: 2024-04-26 05:09:31.199619844 +0000 UTC m=+0.729853665 container remove d528575c5fc77a71ac2ff1803bf37df3af12565be50252413b1ab3fc95c26e8e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T05:09:31.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:09:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:09:31.130Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.5:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.5:8765: connect: connection refused" 2024-04-26T05:09:31.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:09:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:09:31.130Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.5:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.5:8765: connect: connection refused" 2024-04-26T05:09:31.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:09:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:09:31.132Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.5:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.5:8765: connect: connection refused" 2024-04-26T05:09:31.518 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:09:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:09:31.134Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.5:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.5:8765: connect: connection refused" 2024-04-26T05:09:31.519 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:09:31 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:09:31.134Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.5:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.5:8765: connect: connection refused" 2024-04-26T05:09:31.815 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:31 smithi005 podman[145235]: 2024-04-26 05:09:31.46437318 +0000 UTC m=+0.017987208 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:09:31.815 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:31 smithi005 podman[145235]: 2024-04-26 05:09:31.592176173 +0000 UTC m=+0.145790221 container create eae50caa2d63539777e577d3a00ab74a5bbb7b185490f642f6fed4c3e22b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T05:09:31.815 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:31 smithi005 podman[145235]: 2024-04-26 05:09:31.811370097 +0000 UTC m=+0.364984118 container init eae50caa2d63539777e577d3a00ab74a5bbb7b185490f642f6fed4c3e22b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T05:09:31.815 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:31 smithi005 podman[145235]: 2024-04-26 05:09:31.814677941 +0000 UTC m=+0.368291951 container start eae50caa2d63539777e577d3a00ab74a5bbb7b185490f642f6fed4c3e22b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T05:09:32.112 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:31 smithi005 podman[145235]: 2024-04-26 05:09:31.919555817 +0000 UTC m=+0.473169854 container attach eae50caa2d63539777e577d3a00ab74a5bbb7b185490f642f6fed4c3e22b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-04-26T05:09:32.389 INFO:journalctl@ceph.osd.0.smithi005.stdout:Apr 26 05:09:32 smithi005 podman[145235]: 2024-04-26 05:09:32.112751102 +0000 UTC m=+0.666365114 container died eae50caa2d63539777e577d3a00ab74a5bbb7b185490f642f6fed4c3e22b4061 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T05:09:32.756 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T05:09:32.836 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:32.837 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T05:09:32.837 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T05:09:32.837 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.1 2024-04-26T05:09:32.980 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:32 smithi005 systemd[1]: Stopping Ceph osd.1 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:33.480 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[109768]: 2024-04-26T05:09:33.184+0000 7fac6f976700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:33.480 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[109768]: 2024-04-26T05:09:33.184+0000 7fac6f976700 -1 osd.1 963 *** Got signal Terminated *** 2024-04-26T05:09:33.480 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:33 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1[109768]: 2024-04-26T05:09:33.184+0000 7fac6f976700 -1 osd.1 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:09:34.729 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:34.302Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:34.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:34.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:34.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:34 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:34.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:38.869 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:38 smithi005 podman[145355]: 2024-04-26 05:09:38.597889739 +0000 UTC m=+5.525774578 container died d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:09:39.229 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:39 smithi005 podman[145355]: 2024-04-26 05:09:39.042803855 +0000 UTC m=+5.970688725 container cleanup d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5) 2024-04-26T05:09:39.230 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:39 smithi005 bash[145355]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1 2024-04-26T05:09:39.729 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:39 smithi005 podman[145373]: 2024-04-26 05:09:39.319436195 +0000 UTC m=+0.717678972 container remove d589f860de31f4f7d350740da65932ad05b1a739018994ff41feb3798ed057cd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb) 2024-04-26T05:09:40.082 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:39 smithi005 podman[145464]: 2024-04-26 05:09:39.820406088 +0000 UTC m=+0.114691089 container create 326a09fca0cca6397dd976dec25ed6c6a4532a6516be2ebec66da8a3969a3782 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-26T05:09:40.082 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:39 smithi005 podman[145464]: 2024-04-26 05:09:39.724388092 +0000 UTC m=+0.018673104 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:09:40.083 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:40 smithi005 podman[145464]: 2024-04-26 05:09:40.081922879 +0000 UTC m=+0.376207887 container init 326a09fca0cca6397dd976dec25ed6c6a4532a6516be2ebec66da8a3969a3782 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True) 2024-04-26T05:09:40.386 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:40 smithi005 podman[145464]: 2024-04-26 05:09:40.085522225 +0000 UTC m=+0.379807405 container start 326a09fca0cca6397dd976dec25ed6c6a4532a6516be2ebec66da8a3969a3782 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-26T05:09:40.387 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:40 smithi005 podman[145464]: 2024-04-26 05:09:40.156256684 +0000 UTC m=+0.450541688 container attach 326a09fca0cca6397dd976dec25ed6c6a4532a6516be2ebec66da8a3969a3782 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T05:09:40.387 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:40 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:40.157Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:40.387 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:40 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:40.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:40.387 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:40 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:40.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:40.709 INFO:journalctl@ceph.osd.1.smithi005.stdout:Apr 26 05:09:40 smithi005 podman[145464]: 2024-04-26 05:09:40.386441638 +0000 UTC m=+0.680726632 container died 326a09fca0cca6397dd976dec25ed6c6a4532a6516be2ebec66da8a3969a3782 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-26T05:09:41.052 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T05:09:41.094 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:41.095 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T05:09:41.095 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T05:09:41.095 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.2 2024-04-26T05:09:41.422 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:41 smithi005 systemd[1]: Stopping Ceph osd.2 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:41.729 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[115347]: 2024-04-26T05:09:41.420+0000 7ff2ded2b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:41.730 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[115347]: 2024-04-26T05:09:41.420+0000 7ff2ded2b700 -1 osd.2 963 *** Got signal Terminated *** 2024-04-26T05:09:41.730 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:41 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2[115347]: 2024-04-26T05:09:41.420+0000 7ff2ded2b700 -1 osd.2 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:09:44.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:44 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:44.303Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:44.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:44 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:44.304Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:44.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:44 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:44.304Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:47.229 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:46 smithi005 podman[145568]: 2024-04-26 05:09:46.908034904 +0000 UTC m=+5.587301922 container died 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T05:09:47.729 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:47 smithi005 podman[145568]: 2024-04-26 05:09:47.362803715 +0000 UTC m=+6.042070723 container cleanup 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:09:47.730 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:47 smithi005 bash[145568]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2 2024-04-26T05:09:47.730 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:47 smithi005 podman[145589]: 2024-04-26 05:09:47.555873439 +0000 UTC m=+0.647005697 container remove 2deefb6616c40cb3aea3c4279de2b99174c9f6e7edaf53fa32b0db9bc7279c66 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:09:48.317 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:47 smithi005 podman[145681]: 2024-04-26 05:09:47.898427632 +0000 UTC m=+0.018837406 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:09:48.318 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:48 smithi005 podman[145681]: 2024-04-26 05:09:48.031668337 +0000 UTC m=+0.152078099 container create 2188728c62828dc334a7c9d8122c2f87e46f95c2e91affa5d872575172376c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:09:48.318 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:48 smithi005 podman[145681]: 2024-04-26 05:09:48.2339881 +0000 UTC m=+0.354397865 container init 2188728c62828dc334a7c9d8122c2f87e46f95c2e91affa5d872575172376c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T05:09:48.318 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:48 smithi005 podman[145681]: 2024-04-26 05:09:48.237271557 +0000 UTC m=+0.357681332 container start 2188728c62828dc334a7c9d8122c2f87e46f95c2e91affa5d872575172376c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T05:09:48.318 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:48 smithi005 podman[145681]: 2024-04-26 05:09:48.317144495 +0000 UTC m=+0.437554274 container attach 2188728c62828dc334a7c9d8122c2f87e46f95c2e91affa5d872575172376c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, GIT_CLEAN=True) 2024-04-26T05:09:48.730 INFO:journalctl@ceph.osd.2.smithi005.stdout:Apr 26 05:09:48 smithi005 podman[145681]: 2024-04-26 05:09:48.540893835 +0000 UTC m=+0.661303604 container died 2188728c62828dc334a7c9d8122c2f87e46f95c2e91affa5d872575172376c5e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:09:49.135 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T05:09:49.176 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:49.176 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T05:09:49.176 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T05:09:49.176 DEBUG:teuthology.orchestra.run.smithi005:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.3 2024-04-26T05:09:49.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:49 smithi005 systemd[1]: Stopping Ceph osd.3 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:49.979 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:49 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T05:09:49.510+0000 7f9cf4489700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:49.980 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:49 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T05:09:49.510+0000 7f9cf4489700 -1 osd.3 963 *** Got signal Terminated *** 2024-04-26T05:09:49.980 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:49 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T05:09:49.510+0000 7f9cf4489700 -1 osd.3 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:09:50.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:50.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:50.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:50.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:50.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:50 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:50.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:51.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:51 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:51.354+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:52.729 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:52 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T05:09:52.358+0000 7f9ceca75700 -1 osd.3 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.772922+0000 front 2024-04-26T05:09:27.772733+0000 (oldest deadline 2024-04-26T05:09:51.872734+0000) 2024-04-26T05:09:52.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:52 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:52.350+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:53.389 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:53 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:53.108+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:53.693 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:53 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:53.388+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:53.729 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:53 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T05:09:53.328+0000 7f9ceca75700 -1 osd.3 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.772922+0000 front 2024-04-26T05:09:27.772733+0000 (oldest deadline 2024-04-26T05:09:51.872734+0000) 2024-04-26T05:09:53.965 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:53 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:53.692+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:09:54.268 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:54.126+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:54.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3[120380]: 2024-04-26T05:09:54.356+0000 7f9ceca75700 -1 osd.3 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.772922+0000 front 2024-04-26T05:09:27.772733+0000 (oldest deadline 2024-04-26T05:09:51.872734+0000) 2024-04-26T05:09:54.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:54.304Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:54.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:54.305Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:09:54.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:09:54 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:09:54.305Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:09:54.741 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:54.429+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:55.018 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:54 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:54.739+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:09:55.158 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:54 smithi005 podman[145787]: 2024-04-26 05:09:54.830415437 +0000 UTC m=+5.429843268 container died d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T05:09:55.442 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:55 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:55.100+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:55.443 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:09:55 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:09:55.281+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:09:55.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:55 smithi005 podman[145787]: 2024-04-26 05:09:55.331584418 +0000 UTC m=+5.931012291 container cleanup d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-26T05:09:55.480 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:55 smithi005 bash[145787]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3 2024-04-26T05:09:55.724 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:55 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:55.441+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:55.961 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:55 smithi005 podman[145804]: 2024-04-26 05:09:55.524489192 +0000 UTC m=+0.696322144 container remove d52a4be71d7cee844e078365828803415d0c8e37a358955a371f2bd2e07b867c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb) 2024-04-26T05:09:56.018 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:55 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:55.723+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:09:56.211 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:55 smithi005 podman[145899]: 2024-04-26 05:09:55.864047406 +0000 UTC m=+0.018187458 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:09:56.212 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:55 smithi005 podman[145899]: 2024-04-26 05:09:55.975274158 +0000 UTC m=+0.129414240 container create 4ec24aff46fcbf0c1b1408e6aecc9120c51314e9ac73689e254e0f2e2998fcad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-26T05:09:56.212 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:56 smithi005 podman[145899]: 2024-04-26 05:09:56.127990272 +0000 UTC m=+0.282130313 container init 4ec24aff46fcbf0c1b1408e6aecc9120c51314e9ac73689e254e0f2e2998fcad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_CLEAN=True) 2024-04-26T05:09:56.212 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:56 smithi005 podman[145899]: 2024-04-26 05:09:56.131064131 +0000 UTC m=+0.285204179 container start 4ec24aff46fcbf0c1b1408e6aecc9120c51314e9ac73689e254e0f2e2998fcad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-26T05:09:56.212 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:56 smithi005 podman[145899]: 2024-04-26 05:09:56.211331401 +0000 UTC m=+0.365471446 container attach 4ec24aff46fcbf0c1b1408e6aecc9120c51314e9ac73689e254e0f2e2998fcad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:09:56.413 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:09:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:09:56.307+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:09:56.413 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:56.117+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:56.479 INFO:journalctl@ceph.osd.3.smithi005.stdout:Apr 26 05:09:56 smithi005 podman[145899]: 2024-04-26 05:09:56.430780866 +0000 UTC m=+0.584920907 container died 4ec24aff46fcbf0c1b1408e6aecc9120c51314e9ac73689e254e0f2e2998fcad (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:09:56.752 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:56.412+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:57.018 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:56 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:56.751+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:09:57.086 DEBUG:teuthology.orchestra.run.smithi005:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T05:09:57.126 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:09:57.127 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T05:09:57.127 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T05:09:57.127 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.4 2024-04-26T05:09:57.420 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:09:57.321+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:09:57.421 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:57.083+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:57.421 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:57 smithi111 systemd[1]: Stopping Ceph osd.4 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:09:57.421 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:57.419+0000 7fe1ede80700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:09:57.421 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:57.419+0000 7fe1ede80700 -1 osd.4 963 *** Got signal Terminated *** 2024-04-26T05:09:57.422 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:57.419+0000 7fe1ede80700 -1 osd.4 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:09:57.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:57.454+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:58.123 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:57 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:57.775+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:09:58.448 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:58.122+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:58.449 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:09:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:09:58.338+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:09:58.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:58.447+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:09:59.149 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:58 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:58.824+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:09:59.446 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:09:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:09:59.310+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:09:59.446 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:09:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:09:59.149+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:09:59.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:09:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:09:59.444+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:00.193 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:09:59 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:09:59.822+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:10:00.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:00.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:00.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:00.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:00.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:00 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:00.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:00.518 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:00.287+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:00.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:00.423+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:00.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:00.423+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:00.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:00.191+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:00.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:00.191+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:01.180 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:00 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:10:00.794+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:10:01.518 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:01.273+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:01.518 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:01.401+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:01.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:01.401+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:01.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:01.178+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:01.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:01.178+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:02.194 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:10:01.806+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:29.290335+0000 front 2024-04-26T05:09:29.290457+0000 (oldest deadline 2024-04-26T05:09:52.790218+0000) 2024-04-26T05:10:02.194 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:01 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4[35307]: 2024-04-26T05:10:01.806+0000 7fe1e7478700 -1 osd.4 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.692402+0000 front 2024-04-26T05:09:36.692393+0000 (oldest deadline 2024-04-26T05:10:01.392264+0000) 2024-04-26T05:10:02.518 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:02.287+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:02.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:02.419+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:02.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:02.419+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:02.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:02.192+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:02.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:02 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:02.192+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:03.146 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:02 smithi111 podman[99364]: 2024-04-26 05:10:02.871706137 +0000 UTC m=+5.553270540 container died e2994c2a7aa295674a2790ed084dee30dcdb834310d3db08503c88eec3b89cec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T05:10:03.428 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:03 smithi111 podman[99364]: 2024-04-26 05:10:03.244749723 +0000 UTC m=+5.926314133 container cleanup e2994c2a7aa295674a2790ed084dee30dcdb834310d3db08503c88eec3b89cec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-26T05:10:03.428 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:03 smithi111 bash[99364]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4 2024-04-26T05:10:03.429 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:03.330+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:03.429 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:03.330+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:03.429 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:03.220+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:03.429 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:03.220+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:03.768 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:03 smithi111 podman[99384]: 2024-04-26 05:10:03.529621705 +0000 UTC m=+0.653510491 container remove e2994c2a7aa295674a2790ed084dee30dcdb834310d3db08503c88eec3b89cec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:10:03.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:03.427+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:03.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:03 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:03.427+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:04.177 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:03 smithi111 podman[99477]: 2024-04-26 05:10:03.776721805 +0000 UTC m=+0.018458573 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:10:04.177 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:03 smithi111 podman[99477]: 2024-04-26 05:10:03.880475725 +0000 UTC m=+0.122212493 container create 59fc2a46f3b07bdd81a76a6232cd38f5d5cf3f605364829185d5fd2aee400a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:10:04.177 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:04 smithi111 podman[99477]: 2024-04-26 05:10:04.094173015 +0000 UTC m=+0.335909786 container init 59fc2a46f3b07bdd81a76a6232cd38f5d5cf3f605364829185d5fd2aee400a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-26T05:10:04.177 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:04 smithi111 podman[99477]: 2024-04-26 05:10:04.097757223 +0000 UTC m=+0.339493992 container start 59fc2a46f3b07bdd81a76a6232cd38f5d5cf3f605364829185d5fd2aee400a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS) 2024-04-26T05:10:04.178 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:04 smithi111 podman[99477]: 2024-04-26 05:10:04.176882217 +0000 UTC m=+0.418618994 container attach 59fc2a46f3b07bdd81a76a6232cd38f5d5cf3f605364829185d5fd2aee400a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:10:04.473 INFO:journalctl@ceph.osd.4.smithi111.stdout:Apr 26 05:10:04 smithi111 podman[99477]: 2024-04-26 05:10:04.379882262 +0000 UTC m=+0.621619042 container died 59fc2a46f3b07bdd81a76a6232cd38f5d5cf3f605364829185d5fd2aee400a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-26T05:10:04.473 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:04.367+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:04.473 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:04.367+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:04.474 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:04.259+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:04.474 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:04.259+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:04.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:04.305Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:04.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:04.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:04.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:04 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:04.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:04.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:04.471+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:04.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:04 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:04.471+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:04.954 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T05:10:04.992 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:04.992 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T05:10:04.992 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T05:10:04.993 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.5 2024-04-26T05:10:05.259 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:05 smithi111 systemd[1]: Stopping Ceph osd.5 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:05.518 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:05.438+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:05.518 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:05.438+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:05.257+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:05.257+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:05.325+0000 7f63e7a01700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:05.325+0000 7f63e7a01700 -1 osd.5 963 *** Got signal Terminated *** 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:05.325+0000 7f63e7a01700 -1 osd.5 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:05.350+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:05.519 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:05 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:05.350+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:06.518 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:06.284+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:06.518 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:06.284+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:06.518 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:06.357+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:06.519 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:06.357+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:06.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:06.488+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:06.519 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:06 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:06.488+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:07.768 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:07.331+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:07.768 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:07.331+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:07.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:07.501+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:07.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:07.501+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:07.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:07.326+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:07.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:07 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:07.326+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:08.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:08.365+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:08.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:08.365+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:08.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:08.365+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:08.769 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:08.376+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:08.769 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:08.376+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:08.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:08.528+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:08.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:08 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:08.528+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:09.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:09.347+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:09.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:09.347+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:09.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:09.347+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:09.769 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:09.351+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:27.794604+0000 front 2024-04-26T05:09:27.794673+0000 (oldest deadline 2024-04-26T05:09:54.794793+0000) 2024-04-26T05:10:09.769 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5[40876]: 2024-04-26T05:10:09.351+0000 7f63e5001700 -1 osd.5 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:37.695879+0000 front 2024-04-26T05:09:37.695974+0000 (oldest deadline 2024-04-26T05:10:02.995846+0000) 2024-04-26T05:10:09.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:09.572+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:09.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:09 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:09.572+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:10.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:10.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:10.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:10.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:10.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:10 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:10.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:10.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:10.328+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:10.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:10.328+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:10.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:10.328+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:10.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:10.555+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:10.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:10.555+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:10.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:10 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:10.555+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:11.110 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:10 smithi111 podman[99582]: 2024-04-26 05:10:10.825315304 +0000 UTC m=+5.607230403 container died 5da2033307673fecf4ed0ef15fd2d615c89644465e6a7e93c2723feb6bd3336c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-26T05:10:11.380 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:11 smithi111 podman[99582]: 2024-04-26 05:10:11.209553766 +0000 UTC m=+5.991468899 container cleanup 5da2033307673fecf4ed0ef15fd2d615c89644465e6a7e93c2723feb6bd3336c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T05:10:11.380 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:11 smithi111 bash[99582]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5 2024-04-26T05:10:11.768 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:11 smithi111 podman[99611]: 2024-04-26 05:10:11.469382801 +0000 UTC m=+0.647689574 container remove 5da2033307673fecf4ed0ef15fd2d615c89644465e6a7e93c2723feb6bd3336c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:10:11.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:11.588+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:11.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:11.588+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:11.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:11.588+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:11.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:11.378+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:11.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:11.378+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:11.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:11 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:11.378+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:12.098 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:11 smithi111 podman[99702]: 2024-04-26 05:10:11.684058528 +0000 UTC m=+0.018087536 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:10:12.098 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:11 smithi111 podman[99702]: 2024-04-26 05:10:11.803493512 +0000 UTC m=+0.137522491 container create c01450d9f44270c62e2923eebbfc15243132ce6d8189ef66fff37f38ace0fe68 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-26T05:10:12.098 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:12 smithi111 podman[99702]: 2024-04-26 05:10:12.014238839 +0000 UTC m=+0.348267820 container init c01450d9f44270c62e2923eebbfc15243132ce6d8189ef66fff37f38ace0fe68 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:10:12.098 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:12 smithi111 podman[99702]: 2024-04-26 05:10:12.017270837 +0000 UTC m=+0.351299818 container start c01450d9f44270c62e2923eebbfc15243132ce6d8189ef66fff37f38ace0fe68 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-26T05:10:12.098 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:12 smithi111 podman[99702]: 2024-04-26 05:10:12.097595537 +0000 UTC m=+0.431624531 container attach c01450d9f44270c62e2923eebbfc15243132ce6d8189ef66fff37f38ace0fe68 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T05:10:12.397 INFO:journalctl@ceph.osd.5.smithi111.stdout:Apr 26 05:10:12 smithi111 podman[99702]: 2024-04-26 05:10:12.299023005 +0000 UTC m=+0.633051990 container died c01450d9f44270c62e2923eebbfc15243132ce6d8189ef66fff37f38ace0fe68 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:10:12.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:12 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:12.587+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:12.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:12 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:12.587+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:12.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:12 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:12.587+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:12.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:12 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:12.395+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:12.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:12 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:12.395+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:12.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:12 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:12.395+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:12.898 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T05:10:12.935 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:12.935 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T05:10:12.936 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T05:10:12.936 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.6 2024-04-26T05:10:13.268 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 systemd[1]: Stopping Ceph osd.6 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:13.582 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:13.425+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:13.582 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:13.425+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:13.582 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:13.425+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:13.583 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:13.283+0000 7fe0a1cd3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:13.583 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:13.283+0000 7fe0a1cd3700 -1 osd.6 963 *** Got signal Terminated *** 2024-04-26T05:10:13.583 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:13.283+0000 7fe0a1cd3700 -1 osd.6 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:10:13.583 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:13.580+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:13.583 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:13.580+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:13.583 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:13 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:13.580+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:14.729 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:14.305Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:14.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:14.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:14.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:14 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:14.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:14.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:14.626+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:14.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:14.626+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:14.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:14.626+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:14.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:14.452+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:14.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:14.452+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:14.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:14 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:14.452+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:15.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:15.623+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:15.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:15.623+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:15.769 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:15.623+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:15.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:15.501+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:15.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:15.501+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:15.770 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:15 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:15.501+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:16.610 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:16.503+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:16.610 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:16.503+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:16.611 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:16.503+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:17.018 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:16.610+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:17.018 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:16.610+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:17.018 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:16 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:16.610+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:17.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:17.656+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:25.533634+0000 front 2024-04-26T05:09:25.533694+0000 (oldest deadline 2024-04-26T05:09:50.833703+0000) 2024-04-26T05:10:17.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:17.656+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:36.035382+0000 front 2024-04-26T05:09:36.035469+0000 (oldest deadline 2024-04-26T05:09:59.535327+0000) 2024-04-26T05:10:17.768 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6[46649]: 2024-04-26T05:10:17.656+0000 7fe09bacc700 -1 osd.6 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.337185+0000 front 2024-04-26T05:09:45.337195+0000 (oldest deadline 2024-04-26T05:10:10.537326+0000) 2024-04-26T05:10:17.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:17.513+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:17.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:17.513+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:17.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:17 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:17.513+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:18.806 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:18.554+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:18.806 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:18.554+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:18.806 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:18 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:18.554+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:19.080 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:18 smithi111 podman[99807]: 2024-04-26 05:10:18.805652368 +0000 UTC m=+5.647896092 container died b7798f2bbff0e22a057c08801224721ce02890f2029ec8e15a119868488d5bed (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:10:19.422 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 podman[99807]: 2024-04-26 05:10:19.179344931 +0000 UTC m=+6.021588656 container cleanup b7798f2bbff0e22a057c08801224721ce02890f2029ec8e15a119868488d5bed (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux ) 2024-04-26T05:10:19.422 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 bash[99807]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6 2024-04-26T05:10:19.755 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 podman[99824]: 2024-04-26 05:10:19.422048934 +0000 UTC m=+0.613731819 container remove b7798f2bbff0e22a057c08801224721ce02890f2029ec8e15a119868488d5bed (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T05:10:19.755 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:19.552+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:19.755 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:19.552+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:19.755 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:19.552+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:19.756 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:19 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:19.552+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:20.018 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 podman[99919]: 2024-04-26 05:10:19.657285056 +0000 UTC m=+0.018092814 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:10:20.019 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 podman[99919]: 2024-04-26 05:10:19.756234681 +0000 UTC m=+0.117042475 container create ed23fee9e5070a28dd64bf51dea12b4f284318e5cb81039df28ae9c91aa88c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T05:10:20.019 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 podman[99919]: 2024-04-26 05:10:19.975745188 +0000 UTC m=+0.336552936 container init ed23fee9e5070a28dd64bf51dea12b4f284318e5cb81039df28ae9c91aa88c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-26T05:10:20.019 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:19 smithi111 podman[99919]: 2024-04-26 05:10:19.978862407 +0000 UTC m=+0.339670158 container start ed23fee9e5070a28dd64bf51dea12b4f284318e5cb81039df28ae9c91aa88c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-26T05:10:20.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:20 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:20.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:20.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:20 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:20.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:20.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:20 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:20.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:20.518 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:20 smithi111 podman[99919]: 2024-04-26 05:10:20.059086914 +0000 UTC m=+0.419894678 container attach ed23fee9e5070a28dd64bf51dea12b4f284318e5cb81039df28ae9c91aa88c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T05:10:20.518 INFO:journalctl@ceph.osd.6.smithi111.stdout:Apr 26 05:10:20 smithi111 podman[99919]: 2024-04-26 05:10:20.264891301 +0000 UTC m=+0.625699053 container died ed23fee9e5070a28dd64bf51dea12b4f284318e5cb81039df28ae9c91aa88c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-26T05:10:20.820 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:20.587+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:20.821 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:20.587+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:20.821 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:20.587+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:20.821 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:20 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:20.587+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:20.841 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T05:10:20.880 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:20.880 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T05:10:20.880 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T05:10:20.880 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.7 2024-04-26T05:10:21.199 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:20 smithi111 systemd[1]: Stopping Ceph osd.7 for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:21.518 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.198+0000 7f1212110700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:21.518 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.198+0000 7f1212110700 -1 osd.7 963 *** Got signal Terminated *** 2024-04-26T05:10:21.518 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.198+0000 7f1212110700 -1 osd.7 963 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:10:22.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.553+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:22.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.553+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:22.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.553+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:22.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:21 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:21.553+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:22.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:22.523+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:22.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:22.523+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:22.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:22.523+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:22.769 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:22 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:22.523+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:23.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:23.557+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:23.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:23.557+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:23.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:23.557+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:23.768 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:23 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:23.557+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:24.729 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:24.306Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:24.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:24.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:24.730 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:24 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:24.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:25.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:24 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:24.579+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:25.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:24 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:24.579+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:25.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:24 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:24.579+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:25.019 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:24 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:24.579+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:26.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:25 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:25.614+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6806 osd.0 since back 2024-04-26T05:09:26.998094+0000 front 2024-04-26T05:09:26.998361+0000 (oldest deadline 2024-04-26T05:09:52.898149+0000) 2024-04-26T05:10:26.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:25 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:25.614+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6814 osd.1 since back 2024-04-26T05:09:38.000166+0000 front 2024-04-26T05:09:38.000106+0000 (oldest deadline 2024-04-26T05:09:59.600446+0000) 2024-04-26T05:10:26.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:25 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:25.614+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6822 osd.2 since back 2024-04-26T05:09:45.202842+0000 front 2024-04-26T05:09:45.202755+0000 (oldest deadline 2024-04-26T05:10:07.502691+0000) 2024-04-26T05:10:26.018 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:25 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:25.614+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.5:6830 osd.3 since back 2024-04-26T05:09:54.504164+0000 front 2024-04-26T05:09:54.504180+0000 (oldest deadline 2024-04-26T05:10:19.204042+0000) 2024-04-26T05:10:26.019 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:25 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7[52778]: 2024-04-26T05:10:25.614+0000 7f120ff11700 -1 osd.7 963 heartbeat_check: no reply from 172.21.15.111:6804 osd.4 since back 2024-04-26T05:09:59.204545+0000 front 2024-04-26T05:09:59.204629+0000 (oldest deadline 2024-04-26T05:10:25.104688+0000) 2024-04-26T05:10:26.859 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:26 smithi111 podman[100025]: 2024-04-26 05:10:26.608345886 +0000 UTC m=+5.506508592 container died 60f1bafb36a67506b273a4c1f17caf33a471823caf14a1ee2634a4d7f1ba2cdd (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T05:10:27.158 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:26 smithi111 podman[100025]: 2024-04-26 05:10:26.956847972 +0000 UTC m=+5.855010672 container cleanup 60f1bafb36a67506b273a4c1f17caf33a471823caf14a1ee2634a4d7f1ba2cdd (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD) 2024-04-26T05:10:27.158 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:26 smithi111 bash[100025]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7 2024-04-26T05:10:27.518 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:27 smithi111 podman[100054]: 2024-04-26 05:10:27.158105263 +0000 UTC m=+0.545472335 container remove 60f1bafb36a67506b273a4c1f17caf33a471823caf14a1ee2634a4d7f1ba2cdd (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c) 2024-04-26T05:10:27.861 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:27 smithi111 podman[100148]: 2024-04-26 05:10:27.480132527 +0000 UTC m=+0.018038693 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:10:27.861 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:27 smithi111 podman[100148]: 2024-04-26 05:10:27.584036058 +0000 UTC m=+0.121942212 container create bf655d2baaaa140155e3978a867e157068f5e88b8624b608cf1a02b147e0e290 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-26T05:10:27.862 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:27 smithi111 podman[100148]: 2024-04-26 05:10:27.779019194 +0000 UTC m=+0.316925349 container init bf655d2baaaa140155e3978a867e157068f5e88b8624b608cf1a02b147e0e290 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:10:27.862 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:27 smithi111 podman[100148]: 2024-04-26 05:10:27.782206935 +0000 UTC m=+0.320113091 container start bf655d2baaaa140155e3978a867e157068f5e88b8624b608cf1a02b147e0e290 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-26T05:10:27.862 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:27 smithi111 podman[100148]: 2024-04-26 05:10:27.861276671 +0000 UTC m=+0.399182835 container attach bf655d2baaaa140155e3978a867e157068f5e88b8624b608cf1a02b147e0e290 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-26T05:10:28.268 INFO:journalctl@ceph.osd.7.smithi111.stdout:Apr 26 05:10:28 smithi111 podman[100148]: 2024-04-26 05:10:28.063797135 +0000 UTC m=+0.601703290 container died bf655d2baaaa140155e3978a867e157068f5e88b8624b608cf1a02b147e0e290 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T05:10:28.627 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T05:10:28.663 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:28.663 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T05:10:28.664 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-26T05:10:28.664 DEBUG:teuthology.orchestra.run.smithi111:> sudo systemctl stop ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a 2024-04-26T05:10:29.001 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:28 smithi111 systemd[1]: Stopping Ceph prometheus.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:29.268 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:10:29.268 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T05:10:29.268 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T05:10:29.269 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T05:10:29.269 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T05:10:29.269 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T05:10:29.269 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T05:10:29.269 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T05:10:29.269 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.001Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T05:10:29.270 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.002Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T05:10:29.270 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.002Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T05:10:29.270 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a[75372]: ts=2024-04-26T05:10:29.002Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T05:10:29.270 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 podman[100254]: 2024-04-26 05:10:29.011873324 +0000 UTC m=+0.119863004 container died 9e5fc4ef6c62f89a66c0849c803a561da0d735e51bdd87b1be8754fdf519b462 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:29.636 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 podman[100254]: 2024-04-26 05:10:29.376148411 +0000 UTC m=+0.484138085 container cleanup 9e5fc4ef6c62f89a66c0849c803a561da0d735e51bdd87b1be8754fdf519b462 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:29.636 INFO:journalctl@ceph.prometheus.a.smithi111.stdout:Apr 26 05:10:29 smithi111 bash[100254]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T05:10:29.795 DEBUG:teuthology.orchestra.run.smithi111:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-26T05:10:29.828 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:29.828 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-26T05:10:29.828 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:10:30.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:30.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:30.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:30.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi005.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.5:8443: connect: connection refused" 2024-04-26T05:10:30.480 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:30 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:30.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi111:8443/api/prometheus_receiver\": dial tcp 172.21.15.111:8443: connect: connection refused" 2024-04-26T05:10:31.479 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:31 smithi005 systemd[1]: Stopping Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:31.917 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:31 smithi005 ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a[82115]: ts=2024-04-26T05:10:31.607Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:10:31.918 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:31 smithi005 podman[146310]: 2024-04-26 05:10:31.648536509 +0000 UTC m=+0.124026259 container died 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:32.229 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 podman[146310]: 2024-04-26 05:10:32.10811592 +0000 UTC m=+0.583605715 container cleanup 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:32.229 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 bash[146310]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T05:10:32.671 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 podman[146324]: 2024-04-26 05:10:32.400875265 +0000 UTC m=+0.743682881 container remove 3afeca47873055f769e23f99387abdffe0b957f56a76c028403ec049cdfaef7f (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:32.671 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 podman[146324]: 2024-04-26 05:10:32.476383733 +0000 UTC m=+0.819191349 volume remove afa5c254e90b90d09fd308b988832712c8e35e51b48a9e93d3f9c0905bda2792 2024-04-26T05:10:32.979 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:32 smithi005 systemd[1]: Stopping Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:32.980 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T05:10:32.980 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 systemd[1]: Stopped Ceph alertmanager.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T05:10:32.980 INFO:journalctl@ceph.alertmanager.a.smithi005.stdout:Apr 26 05:10:32 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 2.282s CPU time. 2024-04-26T05:10:33.437 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 podman[146430]: 2024-04-26 05:10:33.139568078 +0000 UTC m=+0.114182119 container died c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:33.437 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 podman[146430]: 2024-04-26 05:10:33.437165621 +0000 UTC m=+0.411779658 container cleanup c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:33.729 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 bash[146430]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T05:10:33.730 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 podman[146444]: 2024-04-26 05:10:33.646525393 +0000 UTC m=+0.511318675 container remove c74ab514255d3d4085847072a8705a19968e5060ed5e2b741b4b8bffde42a4fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:10:33.730 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:10:34.225 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T05:10:34.226 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 systemd[1]: Stopped Ceph node-exporter.a for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T05:10:34.226 INFO:journalctl@ceph.node-exporter.a.smithi005.stdout:Apr 26 05:10:33 smithi005 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.118s CPU time. 2024-04-26T05:10:43.426 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:10:45.298 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:45 smithi111 systemd[1]: Stopping Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968... 2024-04-26T05:10:45.564 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:45 smithi111 podman[100703]: 2024-04-26 05:10:45.297973042 +0000 UTC m=+0.111663505 container died cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:10:45.922 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:45 smithi111 podman[100703]: 2024-04-26 05:10:45.662756142 +0000 UTC m=+0.476446606 container cleanup cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:10:45.922 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:45 smithi111 bash[100703]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T05:10:46.263 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:45 smithi111 podman[100717]: 2024-04-26 05:10:45.922123744 +0000 UTC m=+0.628449108 container remove cc0664bb12c9888f53254e72ace59e1564f8af5dd7077d81f7acf5c631efcd31 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:10:46.263 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:45 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:10:46.263 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:46 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T05:10:46.264 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:46 smithi111 systemd[1]: Stopped Ceph node-exporter.b for b5a9bf3e-0386-11ef-bc93-c7b262605968. 2024-04-26T05:10:46.264 INFO:journalctl@ceph.node-exporter.b.smithi111.stdout:Apr 26 05:10:46 smithi111 systemd[1]: ceph-b5a9bf3e-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.764s CPU time. 2024-04-26T05:10:51.283 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:10:51.316 DEBUG:teuthology.orchestra.run.smithi111:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:10:51.342 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T05:10:51.344 DEBUG:teuthology.misc:Transferring archived files from smithi005:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/remote/smithi005/crash 2024-04-26T05:10:51.344 DEBUG:teuthology.orchestra.run.smithi005:> sudo tar c -f - -C /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:10:51.399 INFO:teuthology.orchestra.run.smithi005.stderr:tar: /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:10:51.400 INFO:teuthology.orchestra.run.smithi005.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:10:51.401 DEBUG:teuthology.misc:Transferring archived files from smithi111:/var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/remote/smithi111/crash 2024-04-26T05:10:51.402 DEBUG:teuthology.orchestra.run.smithi111:> sudo tar c -f - -C /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:10:51.437 INFO:teuthology.orchestra.run.smithi111.stderr:tar: /var/lib/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:10:51.437 INFO:teuthology.orchestra.run.smithi111.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:10:51.438 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T05:10:51.439 DEBUG:teuthology.orchestra.run.smithi005:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T05:10:51.472 INFO:teuthology.orchestra.run.smithi005.stdout:1714106565.4422605 mon.a (mon.0) 520 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T05:10:51.473 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-26T05:10:51.473 INFO:tasks.cephadm:Compressing logs... 2024-04-26T05:10:51.473 DEBUG:teuthology.orchestra.run.smithi005:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:10:51.521 DEBUG:teuthology.orchestra.run.smithi111:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:10:51.546 INFO:teuthology.orchestra.run.smithi005.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-26T05:10:51.547 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:10:51.547 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:10:51.547 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T05:10:51.547 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:10:51.547 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T05:10:51.549 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T05:10:51.549 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-26T05:10:51.550 INFO:teuthology.orchestra.run.smithi111.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:10:51.550 INFO:teuthology.orchestra.run.smithi111.stderr:: No such file or directory 2024-04-26T05:10:51.550 INFO:teuthology.orchestra.run.smithi111.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:10:51.550 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:10:51.550 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T05:10:51.552 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T05:10:51.552 INFO:teuthology.orchestra.run.smithi111.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T05:10:51.552 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-26T05:10:51.553 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T05:10:51.558 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.1.log: /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mgr.y.log: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:10:51.558 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T05:10:51.558 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-26T05:10:51.558 INFO:teuthology.orchestra.run.smithi005.stderr: 91.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:10:51.559 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-26T05:10:51.559 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi005.zdnlsb.log 2024-04-26T05:10:51.559 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.a.log: 93.6% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:10:51.559 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:10:51.559 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi005.zdnlsb.log: 81.8% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi005.zdnlsb.log.gz 2024-04-26T05:10:51.559 INFO:teuthology.orchestra.run.smithi005.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:10:51.560 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.cephadm.log: 90.0% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:10:51.564 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mgr.x.log: /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.6.log: 89.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi111.furhfd.log 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr: 85.8% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.cephadm.log: /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi111.furhfd.log: 62.6% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi111.furhfd.log.gz 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr:gzip -5 --verbose -- /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.b.log 83.4% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:10:51.565 INFO:teuthology.orchestra.run.smithi111.stderr: 2024-04-26T05:10:51.566 INFO:teuthology.orchestra.run.smithi111.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:10:51.571 INFO:teuthology.orchestra.run.smithi111.stderr: 91.4% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:10:51.580 INFO:teuthology.orchestra.run.smithi005.stderr:/var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-volume.log: 91.1% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:10:51.601 INFO:teuthology.orchestra.run.smithi111.stderr: 89.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-26T05:10:52.010 INFO:teuthology.orchestra.run.smithi005.stderr: 88.9% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-26T05:10:52.083 INFO:teuthology.orchestra.run.smithi111.stderr: 92.2% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-26T05:10:52.203 INFO:teuthology.orchestra.run.smithi005.stderr: 91.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-26T05:10:53.259 INFO:teuthology.orchestra.run.smithi005.stderr: 90.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-26T05:10:58.295 INFO:teuthology.orchestra.run.smithi111.stderr: 94.4% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T05:10:58.346 INFO:teuthology.orchestra.run.smithi111.stderr: 94.6% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T05:10:59.634 INFO:teuthology.orchestra.run.smithi111.stderr: 94.5% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T05:10:59.899 INFO:teuthology.orchestra.run.smithi005.stderr: 94.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T05:11:00.403 INFO:teuthology.orchestra.run.smithi111.stderr: 94.7% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T05:11:00.405 INFO:teuthology.orchestra.run.smithi111.stderr: 2024-04-26T05:11:00.406 INFO:teuthology.orchestra.run.smithi111.stderr:real 0m8.866s 2024-04-26T05:11:00.406 INFO:teuthology.orchestra.run.smithi111.stderr:user 0m29.942s 2024-04-26T05:11:00.406 INFO:teuthology.orchestra.run.smithi111.stderr:sys 0m1.121s 2024-04-26T05:11:00.522 INFO:teuthology.orchestra.run.smithi005.stderr: 94.5% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T05:11:00.913 INFO:teuthology.orchestra.run.smithi005.stderr: 94.8% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T05:11:01.094 INFO:teuthology.orchestra.run.smithi005.stderr: 95.0% -- replaced with /var/log/ceph/b5a9bf3e-0386-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T05:11:01.096 INFO:teuthology.orchestra.run.smithi005.stderr: 2024-04-26T05:11:01.096 INFO:teuthology.orchestra.run.smithi005.stderr:real 0m9.562s 2024-04-26T05:11:01.096 INFO:teuthology.orchestra.run.smithi005.stderr:user 0m37.747s 2024-04-26T05:11:01.097 INFO:teuthology.orchestra.run.smithi005.stderr:sys 0m1.303s 2024-04-26T05:11:01.097 INFO:tasks.cephadm:Archiving logs... 2024-04-26T05:11:01.097 DEBUG:teuthology.misc:Transferring archived files from smithi005:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/remote/smithi005/log 2024-04-26T05:11:01.098 DEBUG:teuthology.orchestra.run.smithi005:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:11:05.887 DEBUG:teuthology.misc:Transferring archived files from smithi111:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/remote/smithi111/log 2024-04-26T05:11:05.888 DEBUG:teuthology.orchestra.run.smithi111:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:11:09.952 INFO:tasks.cephadm:Removing cluster... 2024-04-26T05:11:09.953 DEBUG:teuthology.orchestra.run.smithi005:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 --force 2024-04-26T05:11:10.495 DEBUG:teuthology.orchestra.run.smithi111:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b5a9bf3e-0386-11ef-bc93-c7b262605968 --force 2024-04-26T05:11:10.998 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T05:11:10.998 DEBUG:teuthology.orchestra.run.smithi005:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:11:11.015 DEBUG:teuthology.orchestra.run.smithi111:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:11:11.030 INFO:tasks.cephadm:Teardown complete 2024-04-26T05:11:11.031 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-26T05:11:11.043 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-26T05:11:11.044 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:11:11.057 DEBUG:teuthology.orchestra.run.smithi111:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:11:11.143 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T05:11:11.153 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T05:11:11.153 DEBUG:teuthology.orchestra.run.smithi005:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:11:11.156 DEBUG:teuthology.orchestra.run.smithi111:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:11:11.169 INFO:teuthology.orchestra.run.smithi005.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:11:11.170 INFO:teuthology.orchestra.run.smithi111.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:11:11.173 INFO:teuthology.orchestra.run.smithi111.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:11:11.173 INFO:teuthology.orchestra.run.smithi111.stdout:=============================================================================== 2024-04-26T05:11:11.174 INFO:teuthology.orchestra.run.smithi111.stdout:^* hv01.front.sepia.ceph.com 3 6 377 63 +1014us[+1022us] +/- 54ms 2024-04-26T05:11:11.174 INFO:teuthology.orchestra.run.smithi111.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 60 -1303us[-1303us] +/- 73ms 2024-04-26T05:11:11.174 INFO:teuthology.orchestra.run.smithi111.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 -29us[ -29us] +/- 68ms 2024-04-26T05:11:11.174 INFO:teuthology.orchestra.run.smithi111.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:11:11.175 INFO:teuthology.orchestra.run.smithi005.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:11:11.175 INFO:teuthology.orchestra.run.smithi005.stdout:=============================================================================== 2024-04-26T05:11:11.175 INFO:teuthology.orchestra.run.smithi005.stdout:^* hv01.front.sepia.ceph.com 3 6 377 63 +1005us[ +992us] +/- 54ms 2024-04-26T05:11:11.176 INFO:teuthology.orchestra.run.smithi005.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 64 -1318us[-1331us] +/- 73ms 2024-04-26T05:11:11.176 INFO:teuthology.orchestra.run.smithi005.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 59 -45us[ -45us] +/- 68ms 2024-04-26T05:11:11.176 INFO:teuthology.orchestra.run.smithi005.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:11:11.176 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T05:11:11.186 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T05:11:11.186 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T05:11:11.195 DEBUG:teuthology.orchestra.run.smithi005:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:11:11.303 DEBUG:teuthology.orchestra.run.smithi111:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:11:11.372 DEBUG:teuthology.orchestra.run.smithi005:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:11:11.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:11:11.402 DEBUG:teuthology.orchestra.run.smithi111:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:11:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:11:11.428 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T05:11:11.438 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T05:11:11.447 INFO:teuthology.task.internal:Duration was 2351.882580 seconds 2024-04-26T05:11:11.447 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T05:11:11.458 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T05:11:11.458 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:11:11.460 DEBUG:teuthology.orchestra.run.smithi111:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:11:11.497 INFO:teuthology.orchestra.run.smithi005.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:11:11.504 INFO:teuthology.orchestra.run.smithi111.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:11:11.801 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T05:11:11.802 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi005.front.sepia.ceph.com 2024-04-26T05:11:11.802 DEBUG:teuthology.orchestra.run.smithi005:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:11:11.828 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi111.front.sepia.ceph.com 2024-04-26T05:11:11.828 DEBUG:teuthology.orchestra.run.smithi111:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:11:11.866 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T05:11:11.867 DEBUG:teuthology.orchestra.run.smithi005:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:11:11.871 DEBUG:teuthology.orchestra.run.smithi111:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:11:11.934 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T05:11:11.934 DEBUG:teuthology.orchestra.run.smithi005:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:11:11.938 DEBUG:teuthology.orchestra.run.smithi111:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:11:12.431 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T05:11:12.442 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T05:11:12.442 DEBUG:teuthology.orchestra.run.smithi005:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:11:12.467 DEBUG:teuthology.orchestra.run.smithi111:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:11:12.491 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T05:11:12.525 DEBUG:teuthology.orchestra.run.smithi005:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:11:12.527 DEBUG:teuthology.orchestra.run.smithi111:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:11:12.548 INFO:teuthology.orchestra.run.smithi005.stdout:kernel.core_pattern = core 2024-04-26T05:11:12.553 INFO:teuthology.orchestra.run.smithi111.stdout:kernel.core_pattern = core 2024-04-26T05:11:12.570 DEBUG:teuthology.orchestra.run.smithi005:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:11:12.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:11:12.616 DEBUG:teuthology.orchestra.run.smithi111:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:11:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:11:12.630 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T05:11:12.640 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T05:11:12.641 DEBUG:teuthology.misc:Transferring archived files from smithi005:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/remote/smithi005 2024-04-26T05:11:12.641 DEBUG:teuthology.orchestra.run.smithi005:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:11:12.702 DEBUG:teuthology.misc:Transferring archived files from smithi111:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573/remote/smithi111 2024-04-26T05:11:12.703 DEBUG:teuthology.orchestra.run.smithi111:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:11:12.739 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T05:11:12.739 DEBUG:teuthology.orchestra.run.smithi005:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:11:12.742 DEBUG:teuthology.orchestra.run.smithi111:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:11:12.787 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T05:11:12.799 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T05:11:12.799 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T05:11:12.809 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T05:11:12.810 DEBUG:teuthology.orchestra.run.smithi005:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:11:12.812 DEBUG:teuthology.orchestra.run.smithi111:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:11:12.825 INFO:teuthology.orchestra.run.smithi005.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:11 /home/ubuntu/cephtest 2024-04-26T05:11:12.842 INFO:teuthology.orchestra.run.smithi111.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:11 /home/ubuntu/cephtest 2024-04-26T05:11:12.843 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T05:11:12.854 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T05:11:12.874 INFO:teuthology.nuke:Checking targets against current locks 2024-04-26T05:11:12.900 DEBUG:teuthology.nuke:shortname: smithi005 2024-04-26T05:11:12.900 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:11:12.924 DEBUG:teuthology.nuke:shortname: smithi111 2024-04-26T05:11:12.924 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:11:12.928 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi005.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:25:42.611804', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK1FqpcbwxuFR2PR6h4paKtpuL+OgrR0eRI4VMHgod6Ly1CTALJy/DKRXKkuUkYBZb2N+iosSnC+rhv1gPf4ez8='} 2024-04-26T05:11:12.949 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi111.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674573', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:25:42.610383', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCtaZHKumg0BsyIabyssGu8Zd23naBvygtR/DUBfKwLs7FaLfGaYfWH39ViSjc84K2xG8q+ZNaoi/CzKgZDHdgc='} 2024-04-26T05:11:12.987 INFO:teuthology.orchestra.console.smithi005:Power off 2024-04-26T05:11:12.988 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:11:13.014 DEBUG:teuthology.orchestra.console.smithi005:power off output: Chassis Power Control: Down/Off 2024-04-26T05:11:13.015 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:11:13.029 INFO:teuthology.orchestra.console.smithi111:Power off 2024-04-26T05:11:13.029 DEBUG:teuthology.orchestra.console.smithi111:pexpect command: ipmitool -H smithi111.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:11:13.042 DEBUG:teuthology.orchestra.console.smithi005:check power output: Chassis Power is on 2024-04-26T05:11:13.053 DEBUG:teuthology.orchestra.console.smithi111:power off output: Chassis Power Control: Down/Off 2024-04-26T05:11:13.053 DEBUG:teuthology.orchestra.console.smithi111:pexpect command: ipmitool -H smithi111.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:11:13.076 DEBUG:teuthology.orchestra.console.smithi111:check power output: Chassis Power is on 2024-04-26T05:11:17.044 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:11:17.077 DEBUG:teuthology.orchestra.console.smithi111:pexpect command: ipmitool -H smithi111.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:11:17.154 DEBUG:teuthology.orchestra.console.smithi005:check power output: Chassis Power is on 2024-04-26T05:11:17.187 DEBUG:teuthology.orchestra.console.smithi111:check power output: Chassis Power is on 2024-04-26T05:11:21.158 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:11:21.188 DEBUG:teuthology.orchestra.console.smithi111:pexpect command: ipmitool -H smithi111.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:11:21.268 DEBUG:teuthology.orchestra.console.smithi005:check power output: Chassis Power is off 2024-04-26T05:11:21.297 DEBUG:teuthology.orchestra.console.smithi111:check power output: Chassis Power is off 2024-04-26T05:11:21.370 INFO:teuthology.orchestra.console.smithi005:Power off completed 2024-04-26T05:11:21.399 INFO:teuthology.orchestra.console.smithi111:Power off completed 2024-04-26T05:11:21.504 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 2351.8825800418854 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e7761ddcfe7b4502affae7ba5bf6666c status: fail success: false 2024-04-26T05:11:21.504 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T05:11:21.579 INFO:teuthology.run:FAIL